{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch as t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU + Sigmoid + Cross Entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 300)\n",
    "        self.fc2 = nn.Linear(300, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mnist_loader import load_data_shared, vectorized_result\n",
    "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n",
    "                                                                     seed=666,\n",
    "                                                                     train_size=1000,\n",
    "                                                                     vali_size=0,\n",
    "                                                                     test_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(data, net):\n",
    "    with t.no_grad():\n",
    "        #for index in range(test_data[0].shape[0]):\n",
    "            # get the inputs\n",
    "        inputs, labels = t.Tensor(data[0]), data[1]\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = t.max(outputs, 1)\n",
    "\n",
    "        correct = (predicted == t.Tensor(labels)).sum().item()\n",
    "        accuracy = correct / data[0].shape[0]\n",
    "        return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net = Net()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr = 1e-1)\n",
    "loss_scores, train_scores, test_scores = fit(net, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3hVVb7/8fc3nZKEACFAgITepeRQpCiIBZ1hLFguFhBRBqzjODNOd2Z+15nxztgLiIiKBRuMoqPYBZEaQHrvoQZCD5C2fn/kOJeLSQjkJDvnnM/refKQc/bi7O9mhw+btddey5xziIhI8IvwugAREQkMBbqISIhQoIuIhAgFuohIiFCgi4iEiCivdly/fn2Xnp7u1e5FRILSokWL9jnnkkva5lmgp6enk5mZ6dXuRUSCkpltLW2bulxEREKEAl1EJEQo0EVEQoQCXUQkRCjQRURChAJdRCREKNBFREJE0AX6ln3HePTTtcxev4/cvAKvyxERqTY8e7DoXC3fcYhnv9rA019uICrCOK9JIr1a1KNn87r40pKIj4v2ukQREU+YVwtc+Hw+d65Pih45kU/m1gPM35TD/M37WZ51iIIiR4RB+0YJ9Gxel57pdfGl1yU5PjbAlYuIeMfMFjnnfCVuC8ZAP11uXgHfbTvIvM05LNycw5LtBziRXwRAi/q16JFeF196Ej2b16VZ3ZqYWUD2KyJS1coK9KDrcilJzZgo+rSqT59W9QHIKyhi+Y5DLNySQ+aWHGas3M1bmdsBSI6PpUd6Er60uvRIr0v7RvFERQbdrQQRkR8IiUA/XUxUBBlpSWSkJcGFLSkqcmzIPsrCLcVX8Au3HOCj5bsBqBUTSfe07wM+ia7N6lAzJiT/WEQkxIVEl8u52HnwOAu35LBo6wEWbjnAmt2HcQ4iI4xOjRPISKtLz+ZJZKSpH15Eqo+Q70MPhMMn8lm89QCZWw6wYEsOS7cf5GRBcT988/q18KUl0aN5cTdNej31w4uINxTo5yCvoIgVOw/9p4smc2sOB3Pzgf/th++ZXpcezevSrmECkREKeBGpfAr0ACipH37HweMAxMdF4UtLomfzevRqUZfOqYlE60ariFSCkB/lUhUiIow2KfG0SYnnpl5pAOw4eJyFm3OYvzmHBZv389XabABqREeSkZZEr+Z16dWiHl2aJhIbFell+SISBnSFHkD7jp78T8DP27SfNbuPABDrH3XTu0U9ereoR9emdYiJ0hW8iJy9CnW5mFlTYDLQECgCJjjnnjytzU3Ag/6XR4GxzrmlZX1uKAb66Q7m5jF/cw7zNxUH/Gr/SJq46Ah8aXU5v2U9+raqT6fGCRoLLyLlUtFAbwQ0cs4tNrN4YBFwlXNu1Slt+gCrnXMHzOxy4E/OuV5lfW44BPrpvg/4uRv3M3fjftbuKb6Cj4+NoleLuvRpWZ++rerTJqW2RtGISIkq1IfunNsF7PJ/f8TMVgOpwKpT2sw55bfMA5pUqOIQVadmDJd1bMhlHRsCxV008zbt59sN+5mzcR+fr94LFI+i6deqONz7t65PSkKcl2WLSJA4qz50M0sHZgGdnHOHS2nzC6Cdc+72EraNBkYDNGvWLGPr1q3nUHLoyjqQy7cb9jF7w37mbNjH/mN5ALRJqU3/1slc0CaZXs3rEhetG6wi4SogwxbNrDYwE3jYOTetlDYDgeeAfs65/WV9Xjh2uZyNoiLH6t2Hmb1+H9+s38eCLTnkFRQRGxVBz+Z1ubBNMgPaNqBlci11z4iEkQoHuplFAx8CnzjnHiulzXnAv4DLnXPrzvSZCvSzcyK/kPmbc5i5NpuZ6/ayMfsYAE2SajCgbTID2zagT8v61IjR1btIKKvoTVEDXgFynHM/K6VNM+BLYPhp/emlUqBXTNaBXL5em83Xa7OZs3EfuXmFxERF0KdlPQa1a8BF7VNIrVPD6zJFJMAqGuj9gG+A5RQPWwT4LdAMwDk33swmAkOB7zvFC0rb4fcU6IFzsqCQBZtz+HLNXr5cs5et+3OB4sU+LmnfgIs7pNA5NVFdMyIhQI/+hxHnHJv2HeOL1Xv4fPVeMrfkUOSgYUIcl3RI4bKODenVoq6mJhAJUgr0MJZzLI+v1uzl01W7mbkumxP5RSTERXFxhxQu79SI/q3ra9SMSBBRoAsAx/MKmb1hHzNW7OazVbs5fKKAWjGRDGqfwhWdGzGgbbLCXaSaU6DLD+QVFDF3035mrNjFjBW7OZCbT82YSC7pkMKQ8xpzQZtkzTcjUg0p0KVMBYVFzNuUw7+X7+TjFbs5mJtPQlwUV3RuxE+6NqZX83qa712kmlCgS7nlFxYxe8M+PvhuJ5+s3M2xvEJSEmK5smsq13RPpV3DBK9LFAlrCnQ5J8fzCvl89R7e/24HX6/NpqDI0b5RAkO7p3JVt1Tq19ZaqyJVTYEuFbb/6Ek+XLaLaYuzWJp1iKgIY2C7Blzva8rAtsma/lekiijQJaDW7znCu4uzmLZ4B9lHTpIcH8u1GU24wdeU9Pq1vC5PJKQp0KVSFBQW8dXabN5auJ2v1u6lsMjRr1V9buzVjEs6pOjhJZFKoECXSrfn8AneXridNxduZ8fB4yTHxzKsZzNu7NmMhomaz10kUBToUmUKixwz1+3l1blb+XpdNhFmXNYxhZF9m+NLS9J8MiIVVKEVi0TORmSEcVG7FC5ql8K2/bm8Nn8rby7YxkfLd9MpNYGRfZozpEtjPbQkUgl0hS6VLjevgGmLd/DynC1s2HuUlIRYbu3TnBt7NiOxZrTX5YkEFXW5SLXgnOPrddlM/GYT327YT62YSG7qncaofs21bqpIOSnQpdpZufMQE2Zt4oOlO4mKiOCa7qmMHdCStHoa9ihSFgW6VFvbc3KZMGsTb2dup6DIcWWXxtw5sBWtGtT2ujSRakmBLtXe3sMnmDBrE6/P38aJgkKu7NKYewe1pkWygl3kVGUF+hmHGphZUzP7ysxWm9lKM7uvhDZmZk+Z2QYzW2Zm3QNRuISPBglx/P7HHZj94EBGX9CCT1bu4eLHZvLA20vZnpPrdXkiQaE8a4o2Aho55xabWTywCLjKObfqlDZXAPcAVwC9gCedc73K+lxdoUtZso+c5PmZG3l13laKnOOmXmncNbAVyfGaEEzCW4Wu0J1zu5xzi/3fHwFWA6mnNbsSmOyKzQPq+P8hEDknyfGx/P7HHZj5y4Fcm9GUV+dt5cJ/fMUTn68jN6/A6/JEqqWzerrDzNKBbsD80zalAttPeZ3FD0MfMxttZplmlpmdnX12lUpYapgYx9+u6cxn91/AgLbJPPH5egb842veXLCNwiJv7v+IVFflDnQzqw1MBX7mnDt8+uYSfssP/rY55yY453zOOV9ycvLZVSphrUVybZ67KYOpY8+nSVINfj1tOUOens38Tfu9Lk2k2ihXoJtZNMVh/rpzbloJTbKApqe8bgLsrHh5Iv9XRlpdpo7tw9PDunEwN48bJszjrjcWs/Pgca9LE/FceUa5GPAisNo591gpzaYDw/2jXXoDh5xzuwJYp8h/mBlDujTmiwcG8LOLW/PF6j0MenQm42duJK+gyOvyRDxTnlEu/YBvgOXA939bfgs0A3DOjfeH/jPAYCAXGOmcK3MIi0a5SKBsz8nlLx+u4rNVe2jVoDZ/vbozPZvX9boskUqhB4skLHyxeg8PTV9J1oHjDOvZjF9f3o7EGpr8S0JLhYYtigSLQe1T+PT+Cxh9QQveWriNix+byYwVu70uS6TKKNAlpNSMieK3V7Tn/bv6kVw7ljGvLeLeKUs4cCzP69JEKp0CXUJS5yaJvH93X+6/uA0fLd/FJY/P4rNVe7wuS6RSKdAlZEVHRnDfxa2Zfnc/GsTHcsfkTH4zbRnHTupJUwlNCnQJeR0aJ/DeXX0ZO6Alby7czo+e+oYl2w54XZZIwCnQJSzEREXw4OB2vHlHb/ILHdeNn8v4mRsp0vQBEkIU6BJWerWox0f39efSjin8/eM1jHhpAdlHTnpdlkhAKNAl7CTWiObZG7vz8NWdWLA5hx899Q0LNud4XZZIhSnQJSyZGTf1SuO9u/pSKzaKYS/MY+I3m/DqQTuRQFCgS1hr3yiB9+/uyyXtU/jvf6/mrjcWaxSMBC0FuoS9hLhoxt3cnd9e0Y4ZK3YzdNwctu3XsncSfBToIhR3wYy+oCWv3NaTXYdOMOSZ2cxev8/rskTOigJd5BT9Wycz/e6+NEyIY8RLC3h13lavSxIpNwW6yGnS6tVi6p19uLBNMn94bwV/+WCVlruToKBAFylB7dgoXhjuY2TfdCZ9u5mfvpqpxaml2lOgi5QiMsJ4aEhH/nJlR75cs5dhL8xn/1E9hCTVlwJd5AyGn5/O+JszWLPrMEPHzWHr/mNelyRSovKsKTrJzPaa2YpStiea2QdmttTMVprZyMCXKeKtSzs25I07enPweD5Dx81hxY5DXpck8gPluUJ/meK1QktzF7DKOdcFGAA8amYxFS9NpHrJSEvi3TF9iImMYNiEeSzcoukCpHo5Y6A752YBZf3kOiDev1B0bX9b3T2SkNSqQW3eGduH5PhYbnlxPl+t3et1SSL/EYg+9GeA9sBOYDlwn3OuKACfK1Itpdapwdtjzqdlcm3ueCWTGSt2eV2SCBCYQL8M+A5oDHQFnjGzhJIamtloM8s0s8zs7OwA7FrEG/VrxzJldG/Oa5LIXW8s4YOlO70uSSQggT4SmOaKbQA2A+1Kauicm+Cc8znnfMnJyQHYtYh3EuKimTyqFxlpSdz35hKmLc7yuiQJc4EI9G3AIAAzSwHaApsC8Lki1V7t2CheHtmD3i3q8cA7S5m6SKEu3inPsMUpwFygrZllmdkoMxtjZmP8Tf4f0MfMlgNfAA865zSrkYSNmjFRTLq1B31a1uOX7y7l/e92eF2ShKmoMzVwzg07w/adwKUBq0gkCMVFRzJxeA9GvryA+9/6jggzhnRp7HVZEmb0pKhIgNSIieTFET3ISEviZ299xycrd3tdkoQZBbpIANWKjeKlkT3pnJrIPW8s0ZzqUqUU6CIB9v2N0hbJtbhjciaLtuqJUqkaCnSRSlCnZgyTR/WkYWIct760kFU7D3tdkoQBBbpIJWkQH8drt/eidmwUI15aoHVKpdIp0EUqUWqdGrw6qif5hUUMnzSffZpPXSqRAl2kkrVqEM+LI3qw+/AJbn1pAUdO5HtdkoQoBbpIFchIS2LczRms2XWEO19fTH6h5q+TwFOgi1SRgW0b8NdrOvPN+n38eupynNPC0xJYZ3xSVEQC53pfU3YePM4Tn68ntU4cP7+0rdclSQhRoItUsfsGtWbnweM89eUGUpNqcEOPZl6XJCFCgS5SxcyMh6/uzO7DJ/ndv1bQJKkmfVvV97osCQHqQxfxQHRkBM/e2I2WybUZ89oi1u854nVJEgIU6CIeiY+LZtLIHsRFRzLy5YUaoy4VpkAX8VBqnRq8OMLHvqMn+emriziRX+h1SRLEFOgiHjuvSR0ev74ri7Ye4LfTNJxRzp0CXaQauLxzI35xaRumLdnBuJkbvS5HgpRGuYhUE3cNbMX6vUf5nxlraVG/NoM7NfS6JAky5VlTdJKZ7TWzFWW0GWBm35nZSjObGdgSRcKDmfHI0PPo2rQOP3/7O1bv0pS7cnbK0+XyMjC4tI1mVgd4DviJc64jcF1gShMJP3HRkUy4JYP4uCjumJxJzrE8r0uSIHLGQHfOzQLKWnLlRmCac26bv/3eANUmEpYaJMQx4RYfe4+cZOxrizSRl5RbIG6KtgGSzOxrM1tkZsNLa2hmo80s08wys7OzA7BrkdDUpWkdHhnamfmbc/jzByu9LkeCRCBuikYBGcAgoAYw18zmOefWnd7QOTcBmADg8/k0NkukDFd3a8KaXUd4ftYmOjZOZFhPzfkiZQvEFXoWMMM5d8w5tw+YBXQJwOeKhL1fDW5H/9b1+eP7K7TYtJxRIAL9faC/mUWZWU2gF7A6AJ8rEvYiI4xnhnWncZ0ajHltMXsOn/C6JKnGyjNscQowF2hrZllmNsrMxpjZGADn3GpgBrAMWABMdM6VOsRRRM5OYs1oXhju49jJAn766iJOFmh6ACmZefWYsc/nc5mZmZ7sWyQYzVixizGvLeYGX1P+PrQzZuZ1SeIBM1vknPOVtE2P/osEicGdGnH3wFa8lbmd1+dv87ocqYYU6CJB5P5L2jCwbTJ//mAlmVt0k1T+LwW6SBCJjDCe+K9uNEmqyZjXFrP7kG6Syv9SoIsEmcQa0Tx/Swa5eQWMfV03SeV/KdBFglCblHj+eV0Xlmw7yJ8/WOV1OVJNKNBFgtQVnRsxdkBL3pi/jSkLdJNUFOgiQe0Xl7alf+v6PPT+SpZsO+B1OeIxBbpIEIuMMJ4e1o2UxFjGvraYvUd0kzScKdBFglydmjE8f7OPg8fzuPv1JZpuN4wp0EVCQIfGCTwy9DwWbMnhvz/UTdJwpTVFRULElV1TWbHjEC98s5lOqYlc52vqdUlSxXSFLhJCHhzcjj4t6/G791awdPtBr8uRKqZAFwkhUZERPHNjd5JrxzLmtUXsO3rS65KkCinQRUJM3VoxPH9LBjnH8rjz9cW6SRpGFOgiIahTamLxTdLNukkaTnRTVCREXdUtlZU7i2+SdmycyPU9dJM01OkKXSSEPTi4Hf1a1ef3763Qk6RhoDxL0E0ys71mVuaycmbWw8wKzezawJUnIhURFRnxnydJx7y2SGuShrjyXKG/DAwuq4GZRQKPAJ8EoCYRCaCkWjG8MNzHkRPFa5KeyNd0u6HqjIHunJsFnGlplHuAqcDeQBQlIoHVrmECj13fle+2H+R3/1qBV2sJS+WqcB+6maUCVwPjy9F2tJllmllmdnZ2RXctImdhcKeG/Ozi1kxdnMWkb7d4XY5UgkDcFH0CeNA5d8b/xznnJjjnfM45X3JycgB2LSJn496LWnNZxxQe/vcqZq3TRVWoCUSg+4A3zWwLcC3wnJldFYDPFZEAi4gwHru+K21S4rnrjcVszD7qdUkSQBUOdOdcc+dcunMuHXgXuNM5916FKxORSlErNoqJI3zEREZw+yuZHMrN97okCZDyDFucAswF2ppZlpmNMrMxZjam8ssTkcrQJKkmz9+SQdaBXO56YzEFmh4gJJhXd7t9Pp/LzMz0ZN8iUuydzO388t1lDD8/jb9c2cnrcqQczGyRc85X0jY9+i8Sxq7zNWXD3qM8P2sTLZNrM6JPutclSQUo0EXC3K8Gt2PTvmP8+YOVpNWryYC2DbwuSc6R5nIRCXOREcYTN3SlXcME7n5jCWt3H/G6JDlHCnQRoVZsFC/e6qNmTCS3vbyQvZrzJSgp0EUEgEaJNZh0aw8O5OYx6pVMcvMKvC5JzpICXUT+o1NqIk8P68bKnYe4d8p3FBZpzpdgokAXkf9jUPsUHhrSkc9X7+H/fbhKE3kFEY1yEZEfGNEnne05uUycvZnUOjW444IWXpck5aBAF5ES/faK9uw6fIKHP1pNw8Q4hnRp7HVJcgYKdBEpUUSE8eh1Xcg+fJIH3l5KcnwsvVvU87osKYP60EWkVHHRkUwYnkGzejW5Y3Imq3cd9rokKYMCXUTKVKdmDJNv60mtmChGTFrA9pxcr0uSUijQReSMGtepweRRPTmRX8iISQvIOZbndUlSAgW6iJRLm5R4Jt3agx0Hj3PrSws4elIPHlU3CnQRKTdfel2eu6k7K3ce5o5XMjmRf8aVJ6UKKdBF5KwMap/Co9d1Ye6m/dwzZYkWx6hGFOgictau6pbKn4Z04LNVe/jV1GUUaYqAaqE8S9BNMrO9ZrailO03mdky/9ccM+sS+DJFpLq5tW9z7r+4DdMW7+CP01doioBqoDwPFr0MPANMLmX7ZuBC59wBM7scmAD0Ckx5IlKd3TuoFbl5BTw/axM1Y6L4zeXtMDOvywpbZwx059wsM0svY/ucU17OA5pUvCwRCQZmxq8vb0duXiETZm2iRnQk91/SxuuywlagH/0fBXwc4M8UkWrMzPjzTzpyPL+QJ79YT3SkcfdFrb0uKywFLNDNbCDFgd6vjDajgdEAzZo1C9SuRcRjERHGI0PPo7DI8c9P1xEVGcGYC1t6XVbYCUigm9l5wETgcufc/tLaOecmUNzHjs/n0x0UkRASGWH887ouFBQ5/v7xGiLNNO1uFatwoJtZM2AacItzbl3FSxKRYBUZYTx+fReKihwPf7SaIuf4qa7Uq8wZA93MpgADgPpmlgU8BEQDOOfGA38E6gHP+e9uFzjnfJVVsIhUb1GRETz5X10xg799vIZC57hzQCuvywoL5RnlMuwM228Hbg9YRSIS9KIiI3jihq5ERhj/M2MthYWOewbpRmll0wIXIlIpoiIjeOz64lB/9LN1HM8v5JeXtdU49UqkQBeRShMZYfzz2i7ERkXy3Ncbyc0r5KEhHRTqlUSBLiKVKiLC+OvVnagRHcmkbzdzPK+Qv17TmcgIhXqgKdBFpNKZGX/4cXtqxUby9JcbOHIyn8dv6EpsVKTXpYUUBbqIVAkz44FL25JYI5r//vdqjpzIZPzNGdSKVQwFiqbPFZEqdXv/Fvzj2vP4dsM+bpo4X8vZBZACXUSq3HW+poy7OYNVuw5z7fg5Wng6QBToIuKJyzo25LVRvdh35CTXjJvDqp2HvS4p6CnQRcQzPZvX5d2xfYiKMG54fi6z1+/zuqSgpkAXEU+1SYln6tg+pCbV4NaXFvD2wu1elxS0FOgi4rnGdWrwzpjzOb9lPX41dRmPfrpWS9qdAwW6iFQL8XHRTLq1Bzf4mvL0lxu4Z8oSTuQXel1WUNEAUBGpNqIjI/j70M40T67FIzPWsP3AcV64JYMGCXFelxYUdIUuItWKmTHmwpaMvzmDdbuPcOWz37JixyGvywoKCnQRqZYu69iQd8eejwHXjp/D9KU7vS6p2lOgi0i11bFxItPv6Ufn1ETunbKEv3+8hsIi3SwtjQJdRKq1+rVjef323tzUqxnjZ25k5MsLOaDpAkqkQBeRai8mKoKHr+7M367pzLyN+xnyzGz1q5fgjIFuZpPMbK+ZrShlu5nZU2a2wcyWmVn3wJcpIgLDejbj7THnU1jkGDpuDm8t3Kbx6qcozxX6y8DgMrZfDrT2f40GxlW8LBGRknVtWocP7+mHLz2JB6cu5xfvLCM3r8DrsqqFMwa6c24WkFNGkyuBya7YPKCOmTUKVIEiIqerVzuWybf14r5BrZm2JIurnv2WDXuPeF2W5wLRh54KnDr5Qpb/vR8ws9FmlmlmmdnZ2QHYtYiEq8gI4/5L2jD5tp7sP5rHkKe/5e3M7WHdBROIQC9pYcAS/0SdcxOccz7nnC85OTkAuxaRcNe/dTIf3defbs3q8Kt3l3Hfm99x5ES+12V5IhCBngU0PeV1E0BPAIhIlUlJiOPVUb34xaVt+HDZTq546hsWbS2rpzg0BSLQpwPD/aNdegOHnHO7AvC5IiLlFhlh3H1Ra97+6fk4B9eNn8vjn62joLDI69KqTHmGLU4B5gJtzSzLzEaZ2RgzG+Nv8hGwCdgAvADcWWnVioicgS+9Lh/f15+ruqXy5BfrGTp+Lhuzj3pdVpUwr24g+Hw+l5mZ6cm+RSQ8fLB0J79/bwUnCwr5zeXtuaV3GhERJd32Cx5mtsg55ytpm54UFZGQNaRLYz69/wJ6t6jHQ9NXcvOL80N6QWoFuoiEtJSEOF66tQd/u6Yzy7IOcdkTs3h17haKQnCSLwW6iIQ8M2NYz2Z8cv8FZKQl8Yf3VzLshXlsCrG+dQW6iISN1Do1mHxbTx4Z2plVuw4z+MlveParDeSHyEgYBbqIhBUz44Yezfji5xcyqF0D/vHJWoY8PZvF2w54XVqFKdBFJCw1SIhj3M0ZjL85g4O5+QwdN4ff/Ws5h44H71OmCnQRCWuDOzXk8wcuZGSf5kxZsI1Bj37N1EVZQTknjAJdRMJe7dgo/jikA9Pv7keTpJo88M5Srn9+Lqt3Hfa6tLOiQBcR8euUmsi0sX14ZGhnNmYf40dPfcMf3lsRNEveKdBFRE4REVF80/TLBy5k+PnpvLFgGwP++TWvzNlS7UfDKNBFREpQp2YMf/pJRz6+rz+dUxN5aPpKLntiFl+s3lNt+9cV6CIiZWiTEs+ro3oycbgPHIx6JZObJs6vlotUK9BFRM7AzLi4Qwqf3H8BfxrSgVW7DvPjp2dz35tLqtXcMJptUUTkLB0+kc/4rzcy6dvNFBY5buqVxl0DW5EcH1vp+y5rtkUFuojIOdp96ARPfrGOtzOziI2K4La+zbnjghYk1oiutH0q0EVEKtGm7KM89tk6Ply2i/i4KO7o34KRfdOJjwt8sCvQRUSqwKqdh3n883V8tmoPiTWiub1fc0b0TSchgMFe4QUuzGywma01sw1m9usStjczs6/MbImZLTOzKypatIhIsOnQOIEXhvv44O5+ZKQl8ehn6+j39y958vP1VTJHzBmv0M0sElgHXAJkAQuBYc65Vae0mQAscc6NM7MOwEfOufSyPldX6CIS6pZnHeKpL9fz2ao91I6N4pbz07itb/MK3Tyt6BV6T2CDc26Tcy4PeBO48rQ2Dkjwf58I7DzXYkVEQkXnJom8MNzHR/f2Z0DbZMbP3Ei/R75k4jebKmV/UeVokwpsP+V1FtDrtDZ/Aj41s3uAWsDFJX2QmY0GRgM0a9bsbGsVEQlKHRon8MyN3fl59lHGz9xIk6QalbKf8lyhl7RE9un9NMOAl51zTYArgFfN7Aef7Zyb4JzzOed8ycnJZ1+tiEgQa5Fcm/+5tguDOzWqlM8vT6BnAU1Ped2EH3apjALeBnDOzQXigPqBKFBERMqnPIG+EGhtZs3NLAb4L2D6aW22AYMAzKw9xYGeHchCRUSkbGcMdOdcAXA38AmwGnjbObfSzP5iZj/xN3sAuMPMlgJTgFtddZ2OTEQkRJXnpijOuY+Aj05774+nfL8K6BvY0kRE5GxotkURkRChQBcRCREKdBGREKFAFxEJEZ7Ntmhm2cDWc/zt9YF9ASwnWITjcYfjMUN4Hnc4HjOc/XGnOedKfDLTs0CvCOHeox8AAAOtSURBVDPLLG1ymlAWjscdjscM4Xnc4XjMENjjVpeLiEiIUKCLiISIYA30CV4X4JFwPO5wPGYIz+MOx2OGAB53UPahi4jIDwXrFbqIiJxGgS4iEiKCLtDPtGB1KDCzpv5Ft1eb2Uozu8//fl0z+8zM1vt/TfK61spgZpH+Bcc/9L9ubmbz/cf9ln8a55BhZnXM7F0zW+M/5+eHw7k2s/v9P98rzGyKmcWF4rk2s0lmttfMVpzyXonn14o95c+3ZWbW/Wz2FVSB7l+w+lngcqADMMy/KHWoKQAecM61B3oDd/mP89fAF8651sAX/teh6D6Kp2r+3iPA4/7jPkDxgiqh5ElghnOuHdCF4mMP6XNtZqnAvYDPOdcJiKR4rYVQPNcvA4NPe6+083s50Nr/NRoYdzY7CqpAp3wLVgc959wu59xi//dHKP4Lnkrxsb7ib/YKcJU3FVYeM2sC/AiY6H9twEXAu/4mIXXcZpYAXAC8COCcy3POHSQMzjXF03fXMLMooCawixA81865WUDOaW+Xdn6vBCa7YvOAOmZW7vXqgi3QS1qwOtWjWqqEmaUD3YD5QIpzbhcUhz7QwLvKKs0TwK+AIv/resBB/0IrEHrnvAXFq3u95O9mmmhmtQjxc+2c2wH8k+LVznYBh4BFhPa5PlVp57dCGRdsgV6eBatDhpnVBqYCP3POHfa6nspmZj8G9jrnFp36dglNQ+mcRwHdgXHOuW7AMUKse6Uk/j7jK4HmQGOgFsXdDacLpXNdHhX6eQ+2QC/PgtUhwcyiKQ7z151z0/xv7/n+v1/+X/d6VV8l6Qv8xMy2UNyddhHFV+x1/P8th9A751lAlnNuvv/1uxQHfKif64uBzc65bOdcPjAN6ENon+tTlXZ+K5RxwRbo5VmwOuj5+41fBFY75x47ZdN0YIT/+xHA+1VdW2Vyzv3GOdfEOZdO8bn90jl3E/AVcK2/WUgdt3NuN7DdzNr63xoErCLEzzXFXS29zaym/+f9++MO2XN9mtLO73RguH+0S2/g0PddM+XinAuqL+AKYB2wEfid1/VU0jH2o/i/WcuA7/xfV1Dcn/wFsN7/a12va63EP4MBwIf+71sAC4ANwDtArNf1BfhYuwKZ/vP9HpAUDuca+DOwBlgBvArEhuK5BqZQfJ8gn+Ir8FGlnV+Ku1ye9efbcopHAZV7X3r0X0QkRARbl4uIiJRCgS4iEiIU6CIiIUKBLiISIhToIiIhQoEuIhIiFOgiIiHi/wOOoLKPIv6sqAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD6CAYAAACxrrxPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xUVfrH8c+TQAgJoYSEIhAIEJoUgVBUFFBZKQqrooJlFUFE4WfflbWL7tp21dVFV1SUjoKKqCDNihST0ItAqAmBkEIKIX3O7487YAgDmZBJJjPzvF8vXsy998zNcx34cj1z7jlijEEppZTn83N3AUoppVxDA10ppbyEBrpSSnkJDXSllPISGuhKKeUlNNCVUspLOBXoIjJYRHaJSLyITHZwvKWIrBKRLSLyo4g0d32pSimlzkfKGocuIv7AbmAQkAjEAKONMTtKtFkAfGOMmSEiVwFjjDF3nu+8YWFhplWrVhUsXymlfEtcXFyqMSbc0bEaTry/NxBvjNkHICLzgRHAjhJtOgGP2F//ACwq66StWrUiNjbWiR+vlFLqFBE5eK5jznS5NAMSSmwn2veVtBm4yf76BiBERBqWp0illFIV40ygi4N9pftpHgf6i8hGoD9wGCg660Qi40UkVkRiU1JSyl2sUkqpc3Mm0BOBFiW2mwNJJRsYY5KMMTcaY7oDT9n3ZZY+kTFmmjEm2hgTHR7usAtIKaXUBXIm0GOAKBGJFJEAYBSwuGQDEQkTkVPn+jsw3bVlKqWUKkuZgW6MKQImAcuAncBnxpjtIjJFRIbbmw0AdonIbqAx8I9KqlcppdQ5lDlssbJER0cbHeWilFLlIyJxxphoR8f0SVGllPISzoxDV0qpypWyGxJjoNto8KuE+0xjYOtCSN3t+nNfiPaDoVlPl59WA10p5V67l8HCe6DgBOxaAjdOg4Bg152/KB++eRQ2zbbvcDQSu4qFNNFAV0p5EWNg7VRY/jQ07QodroMfX4bp18Lo+VDPBVNC5aTCp3fCoTXQfzIMmAxSDQK9kmigK6UqrrgIPr0dDq51/j3GBgXZ0GkE/Pk96678ou6wYAy83R1q1K54XUV5VoDf9BF0GVnx85VTsc1w/GQB6TkFpJ7IJz3Het0nsiHtm4S4/OdpoCulKi52Ouz+DrreCrVDnX9fWFvoec8f/eZRg+DeVbBhJtiKK16X+EHXm61/KMopO6+QxOO5pOcUkJZTwIm8sx5+B6Cw2HY6qNNy8kk7UXD6PcdPFuBoIOELwy/WQFdKVUMnUuD7lyCyP9zwfsW7NMLbw7WV9yhLVl4hcQeOcyAt53Tw5hX+8Y9H2okC9iRnk5SZV67z1g+qSWhwAGHBtWgTXodekQGEBQfQsE4tQoMDaGh/3bBOAPVr13T1ZQEa6Eqpilr5PBTmwNDXq1X/dObJQmIPppOUkUuavctjS2Im2w5nYrPfNfsJNAgKoHaA/+n31Q2sSe/IUKIah9CqYTAN61hhHBJY0+Hl+YnQIKgmNfzdPwpcA10pdeESfrNGj1z2oHVn7QbHcwrYmHCco5n5pOfkcyw7n7iDx9lxJOuM7o56tWvSvnEIk66Kom/rUNo3DqF+UAD+ftXnH6GK0kBXyhdsnAMxHzjulw6sB4NfhiZdynfOnFT4+iEIuQj6/801dZZQUGRjdXwK6/enY7Od3RGdW1hM3MEMfj96ZnCH1KpB52b1ePjqdvRpHUrr8GAaBAVQsxrcQVc2DXSlvJmtGJY/A+umWoFd18FQwKSN8NG11vjvjtc5d97kHTDvVjhxDG6dA7Vc8wXfsew81u9L58ddKazYcZSsvCJq+ovDMPb3E7o2r8ej17Sjd2QoEQ2DaBAUQGBNfwdn9g0a6Ep5opzUsp96NAZ+fQv2LIc+E+BP/wB/B3/ls4/CvNHw6R0w8Elo1e/85z1+EJY8DgF1YMyScj8gU1BkOz0i5PDxXPYcO8Ge5Gy2Hs5kb0oOAHUDazCoUxOGdW1Cv7bhBNTw/rtrV9BAV8rT7P/ZelgmL6PstuIPw96AXmPP3SakiRXMix6AH5wbXXKsTgeWdnmTnD31CDl0gNBgayTH8ZMF7E7OZs+xEyRlWEP+0k8UkFtiFEmRg+6Ti+oF0qFpXW6JbkHf1g25+KK61eJLRk+jga6UJ4n92Lo7btgWbvwAagScv329FtCwTdnnrVkbRk6Hvg9AYQ65hcX8uCuF7Ycz+T05m5MFfwSyDT82prYl/4fjwPGzTiUCEaFBtGhg/QoNDiAowP/0CJFaNfxPjxxpXDeQto3qEBJYOcP4fI0GulKeoLjIekR+/XvQdpAVvoF1XfszRDDNo1m8OYl/LtlJclYIrcOa0KdrQ/rY+6gbBgfQIDiAgBJ3z1l5hafvxOvWrkmb8DpnDANUVUcDXanqLi/Tehx+7yrocz/86SXHfeHlZLMZdiVnszkhg9QT+aTlFLApIYONhzLo0qwe797eg54ty37qM7CmP41CAitcj6o4DXSlqrP0fTB3FKTvhev/Az3vPm/zomIbmxMzWbcvjXX70kjOyqNNeB2iGofQtF4gGScLSTuRz6H0k/x2IJ2Mk4Wn3xtSqwaN6wXyyo1duDm6hVeNz/YVTgW6iAwG/gP4Ax8aY14pdTwCmAHUt7eZbIxZ4uJalfItB1ZbI08A7lwEkVc4bFZUbGPtvjS+3XKEZduPctwe0u0bh9CiQRC/H81m2fajp5+ODArwp3HdQAZ1bEzf1g2JbtWAJvUCqVVDu0k8XZmBLiL+wFRgEJAIxIjIYmPMjhLNnsZaa/Q9EekELAFaVUK9SvmGuBnw7aMQ2tqaStbBF5uFxTa+2JDIO9/Hk3g8l+AAf67u2JhrL25C39ahNKxT63TbvMJi0nIKCC31mLvyLs7cofcG4o0x+wBEZD4wAigZ6AY49Q1NPSDJlUUq5TNKPgjU5mrry8/a9f84bDPsPJrFmvg0Zqw9QOLxXLq1qM9TQzsysEOjcz5UE1jTn2b1XTAdrarWnAn0ZkBCie1EoE+pNs8Dy0Xk/4Bg4BpHJxKR8cB4gIiIiPLWqpR3y8uCz8ee9SCQMYb1+9OZs/4QP+9OITPX6lLp1qI+L47ozID24Ug1mhRLuY8zge7oT0rpJwNGA58YY/4tIpcCs0SkszHGdsabjJkGTAOIjo52MEuwUj4qfT/MGwWpe04/CFRsMyz47RDTf93P7uQT1Ktdk8EXN+HSNg3p0zqUpvX0jludyZlATwRalNhuztldKmOBwQDGmLUiEgiEAcdcUaRSXu3EMfhoEBQXwp1fQuv+rN+XxvNf72DnkSwuvqgur93Uleu7XaT93+q8nAn0GCBKRCKBw8Ao4LZSbQ4BVwOfiEhHIBBIcWWhSnmtFc9Bbgbc9xOpwW2ZMm8jizcn0ax+bd69vQdDOjfRLhXllDID3RhTJCKTgGVYQxKnG2O2i8gUINYYsxh4DPhARB7B6o652xhHCy8ppc5waB1sngv9HuHb5FCe+epnTuQV8dDVUUzo30bvyFW5iLtyNzo62sTGxrrlZytVLRQXwbQB2HLTeazRB3y5LYNuzevxr5u7EdXY9etNKu8gInHGmGhHx/RJUaXcJXY6JG/lH0FPsHhHJn+9tj33XdlaZxlUF0wDXSl3OLKFopVTiKUrC072YMaYaPpFhbm7KuXhNNCVqmo7v6Z44b2kFtXmneAHWDSmH63D67i7KuUFNNCVqirGwOo3YNUUtpm2vFrvGd6dMJT6QWXMaa6UkzTQlaoqWxfCqiks5XLeCH6QOfcO0DBXLqWBrlRVyMui+Lsn2SVtec7/YRaO7adziCuX06/TlaoCRT+8jJxM4YXiMcwY25eIhkHuLkl5IQ10pSqZSd6OrH+f+UUDGXPLSDo2dfHScUrZaZeLUpUh8zAUFwCQ/OlD1DK1OX7pZG7r3MTNhSlvpoGulCsV5cM3j8CmOad3NQFmNHyICUN6u68u5RM00JVylRMp1pJxCeswl05iVXo4S7YeoV5YUx4e94Cu0akqnQa6Uq6Qthdm/RlOHKPgho944ve2fLn5MMO79eeJkV3PuZKQUq6kga5URRkDi/8P8rI4futX3LOimI2HDvP4n9oxcWBbnfpWVRkNdKUqausCOPgrSVe8ysiFOaSfLOC923swpEtTd1emfIwGulIVkZcFy58mJ6wbg36KICQQFk64jM7N6rm7MuWDNNCVqogfX4ETx/hn4FPUCQzgq0mX07iuPgGq3EMfLFLqQh3dBuv/R1r70cxJDGNcv9Ya5sqtnAp0ERksIrtEJF5EJjs4/qaIbLL/2i0iGa4vValq5OAamDkcajfg1cJbCAmswajeLcp+n1KVqMxAFxF/YCowBOgEjBaRTiXbGGMeMcZcYoy5BHgH+KIyilWqWtg4G2YMh9qhJN64iIU7T3JH35aEBNZ0d2XKxzlzh94biDfG7DPGFADzgRHnaT8amOeK4pSqVmzFsPxp+GoitOoH41byv21Qw8+PMZe1cnd1SjkV6M2AhBLbifZ9ZxGRlkAk8P05jo8XkVgRiU1JSSlvrUq5T14WzL8N1rwDvcfD7QtJLa7NgthEbuzRjEbad66qAWcC3dFTEeYcbUcBC40xxY4OGmOmGWOijTHR4eHhztaolHsdPwjTr4U9K2DYv2Ho6+BfgzdW7Ca/yMa4K1q7u0KlAOeGLSYCJb/taQ4knaPtKGBiRYtSqto4uBY+vR1sRXDH59BmIACfxSQwd/0hxl/ZmraNdD1QVT04c4ceA0SJSKSIBGCF9uLSjUSkPdAAWOvaEpVyk01zrZEsgfVh3KrTYb4pIYOnF22jX9sw/nZtezcXqdQfygx0Y0wRMAlYBuwEPjPGbBeRKSIyvETT0cB8Y8y5umOU8gy2YljxHCy6HyIuhXErISwKgJTsfCbMiqNR3Vq8M7o7Nfz1UQ5VfTj1pKgxZgmwpNS+Z0ttP++6spRyk/wT8MW9sGsJRN8DQ14Df2s4Yl5hMRNmx5GRW8Dn919Gg2Bd4FlVL/rov1KnZByCeaPh2A4Y8jr0vhfsMyXabIa/LtxC3MHjvHt7Dy6+SOdqUdWPBrpSAAm/WcMSiwrg9oXQ9uozDr+5cjdfb07iicEdGKqzKKpqSjsAldo8Hz4ZBgF1YNyKs8J81rqDvPN9PLdGt2BCfx2iqKovvUNXvu3HV+HHf0KrK+CWmRAUevrQyYIinv1qOwvjEhnQPpyXbuisi1Woak0DXfmuQ+utMO96K4yYevrLT4D4Y9ncP3sD8SknePDqKB66OkrXBFXVnga68k22YljyGNRtBsPeOCPMP49L5OlF2wgK8GfmPb25IkqfalaeQQNd+abY6XB0K9z8CdSynvTMLSjmucXb+Cw2kT6Robw9urvOb648iga68j0nUmDVixDZHzr9GbD6y0dPW8eWw5k8eFVbHrw6Sh8aUh5HA135npXPQ2GONcmWCMU2w0PzN7H1cCbv3d6TwZ2buLtCpS6I3oIo35LwG2yaDX0fgHBrHpaXl+xkxY5knr2uk4a58mga6Mp32Irh28cg5CLo/zcAZq87yIer93P3Za24+/JINxeoVMVol4vyHbHT4egWGDkdaoWwbl8azy3ezsD24TxzXaey369UNad36Mo35KTC9y9C5JVw8Y0kZeQycc4GWjYM4j+ju+sYc+UV9A5dea/EWMhMtF5vWwgFOTDkdfKKbEyYHUd+kY1pd0ZTVxd3Vl5CA115pz0rYM7IM/f1exQT3p6nFmxhS2Im0+7sqasNKa+iga68T1E+LP0bNGxrzc+CWE+CNmzLv5fv5vMNiTx0dRR/ulhHtCjv4lQfuogMFpFdIhIvIpPP0eYWEdkhIttFZK5ry1SqHNa8Den7rMUpGl8MjTtBWBQfrt7Pf3+IZ1SvFjx8TZS7q1TK5cq8QxcRf2AqMAhrwegYEVlsjNlRok0U8HfgcmPMcRFpVFkFK3VeGYfg539Dx+FnTIP7eVwiL327k8EXN+EfN3TRWROVV3Kmy6U3EG+M2QcgIvOBEcCOEm3uBaYaY44DGGOOubpQpc4pdQ8UnrRe//CytcrQtf88fXj9vjSe+HwLl7dtyH9GX6IjWpTXcibQmwEJJbYTgT6l2rQDEJFfAX/geWPMdy6pUKnziV8Js286c9/Vz0L9FgAkZeTywJwNRDQM4r07elKrhr8bilSqajgT6I5uZ4yD80QBA4DmwC8i0tkYk3HGiUTGA+MBIiIiyl2sUmeJ+QiCG8H1b1nbAXWsseZYizrfr8MTlQ9xJtATgRYltpsDSQ7arDPGFAL7RWQXVsDHlGxkjJkGTAOIjo4u/Y+CUuWTeRh2fweXPwwdhp1xyBjDM4u2sVmHJyof4swolxggSkQiRSQAGAUsLtVmETAQQETCsLpg9rmyUKXOsnE2GBv0+MtZh975Pp4FcYk8eFVbHZ6ofEaZgW6MKQImAcuAncBnxpjtIjJFRIbbmy0D0kRkB/AD8FdjTFplFa0UtmLYMBPaXAWhZ06qNWvtAd5YsZubejTn4Wvauac+pdzAqQeLjDFLgCWl9j1b4rUBHrX/Uqryxa+ErEQY/M8zdi/enMSzi7dzTcfGvHpTF/x0RIvyITo5l/JMcZ9YX4a2H3p614odyTz66SZ6tQrlv7d11xWHlM/RP/HK82Qcsr4M7X7H6cWdV+5I5oE5cVzcrB4f3hVNYE0dnqh8jwa68izJ2+HjYeBfC3reBVhhfv+cODo1rcvMe3rr8ETlszTQlefY9R189CcoLoAx30KDVqzaWSLMx/ahXm0Nc+W7dLZF5Rl2fgOf3gFNu8Lo+VD3IlbtTGbC7Dg6apgrBWigK0+QfwKW/BWadIYx30FAkHVnPnsDHZvWZZaGuVKABrryBD+/BtlJcMsMCAhi3b407p+9gQ5NQzTMlSpB+9BV9ZayC9ZOhUvugBa9ScnO5//mbaR5aG0Nc6VK0Tt0VX0ZY3W1BATDNc9TbDM88ukmsnILmTW2t4a5UqXoHbqqvmI+hP0/wVXPQJ1w3v0hntXxqbww/GI6NKnr7uqUqnb0Dl1VP7ZiWPkcrHkH2lwN0ffw0+4U3ly5mz9fchG39mpR9jmU8kEa6Mr9igrgyGbAWN0sq9+wngTtPR6ufZlf9qYzfmYs7RqH8JIuH6fUOWmgK/cyBubeDPt+/GOf+MPQf0Hve1m9J5VxM2KJDAtm7r19qVNL/8gqdS76t0O51/YvrDC/4nFoeam1r35LCIti7d40xs6IOR3mocEBbi1VqepOA125T342LHsKmnaDgU+C3x8TasUfy+a+WbFEhAZpmCvlJA105T4/vQbZR+CWWWeEeeqJfMZ8EkNADT+m391Lw1wpJ2mgK/dI2QXr3rWmwG3R6/TuvMJixs+M5VhWPvPH96VFaJAbi1TKszg1Dl1EBovILhGJF5HJDo7fLSIpIrLJ/muc60tVXuOMB4ZeOL07M7eQe2fGsuFQBm/eegndIxq4sUilPE+Zd+gi4g9MBQYBiUCMiCw2xuwo1fRTY8ykSqhReZsdi6wHhob+C4LDANifmsPYGTEkpJ/ktZFdGdqlqZuLVMrzONPl0huIN8bsAxCR+cAIoHSgK1W2/BPw3ZPQpCtE3wPA+n1pjJ8Vh5/A7LF96NO6oZuLVMozOdPl0gxIKLGdaN9X2k0iskVEFoqIw0f5RGS8iMSKSGxKSsoFlKs83s+vWzMnDvs3+PmzZm8qd338G+EhtfhqYj8Nc6UqwJlAd/RYnim1/TXQyhjTFVgJzHB0ImPMNGNMtDEmOjw8vHyVKs+Xusc+c+Lt0KI3a/amcs8nMbQMDWb++L5ENNQvQJWqCGe6XBKBknfczYGkkg2MMWklNj8AXq14acrj2YqtVYb2/WRtFxdAzSC45gXWxKdyz4wYIkKDmHNvH8Lq1HJvrUp5AWcCPQaIEpFI4DAwCritZAMRaWqMOWLfHA7sdGmVyjNtmAG7lkDXUae//KTj9Xy2M4+nFm09/QSohrlSrlFmoBtjikRkErAM8AemG2O2i8gUINYYsxh4UESGA0VAOnB3JdasPEFOGqyaAi37wQ3/AxGKbYZXlu7kg1+20K9tGFNv60G9IJ3TXClXcerBImPMEmBJqX3Plnj9d+Dvri1NebRVL0BeFgx9HUQoKLIxae4Glu9I5q5LW/LMdZ2o4a/T8SvlSvqkqHK9w3GwYSZcOhEad6KgyMYDczawcmcyz13fiTGXR7q7QqW8kga6cq2cVFj8INRpBP2foKDIxsS5Vpi/OOJi7ry0lbsrVMpraaAr10neAfNuhexkuHU2ef7BTLLfmU/RMFeq0mmgK9fYvRwW3gMBQTBmKVlhXbl3+m/8diCdF//cmTv7tnR3hUp5PQ10VTHGWLMmLn8aGneG0fNJ8Qvj7mnr2HU0m7duvYQRlzh6sFgp5Woa6OrCFRXAt4/CxlnQ8Xq44X2OF9bk1v+t4UhGHh/eFc2A9o3cXaVSPkMDXV2YnDT47C9wcLW1fNzAp8i3Ge6b9RuJx3OZPbYPvSND3V2lUj5FA12V37HfrS8/s47AjR9A11swxjD58y38diCdt0d31zBXyg000FXZctKg8KT1+ugW+HIC1AiEMUugeTQAb6+K58uNh3lsUDuGd7vIjcUq5bs00NW5GQM/vQo/vnzm/sZdYPQ8qG/N2fbVpsO8uXI3N/VozqSr2rqhUKUUaKCrcynMhUUPwPYvoPNIaD3A2l+jFnQYZi0fB8QcSOevC7bQJzKUl2/sgoij2ZaVUlVBA11Zjh+AuE+guNDaPvALHNlirfl5+UPgIKgPpOYwfmYszRvU5v07exJQQ+dmUcqdNNAVHFxjzVuemwE1a1v7aoXAqLnQYehZzfOLilm69ShvrNgNwPS7e1E/KKAqK1ZKOaCB7us2zIJvHoEGrWDsCmjY5pxND2fkMnf9Qeb/lkBaTgGtw4L58K5etAoLrrp6lVLnpIHuq2zFsPI5WPMOtB4IN38Cteuf3cxm+HVvKjPXHmTVzmQAru7YmL9c2pLL24Th56d95kpVFxrovigvC764F3Z/B73uhcGvgP+ZfxQycwtZGJfInHUH2ZeaQ2hwABP6t+G2PhE0b6BrfypVHWmg+4KMQ7B7mfXaGIj7GFJ2wdB/Qe97z2iaW1DM+z/v5f2f9pFbWEz3iPq8eWs3hnZpSq0a/m4oXinlLKcCXUQGA//BWoLuQ2PMK+doNxJYAPQyxsS6rEpVMSues4YfnhJYH+74HNoMPL3LGMPSbUf5x7c7OZyRy7AuTbl/QBs6N6vnhoKVUheizEAXEX9gKjAISARiRGSxMWZHqXYhwIPA+sooVF0gY+Dgr9BpBAx7w9oXUAdqBp5ucjQzjye/3Mr3vx+jQ5MQ5o/vS9/WDd1UsFLqQjlzh94biDfG7AMQkfnACGBHqXYvAq8Bj7u0QlUx6fvgRDJE9ofgsDMOGWP4fMNhpny9nYJiG89c14m7Lm2pa30q5aGcCfRmQEKJ7USgT8kGItIdaGGM+UZEzhnoIjIeGA8QERFR/mpV+R1cY/3e8rIzdidn5fHkF1tZ9fsxerVqwGsjuxGpww+V8mjOBLqjcWnm9EERP+BN4O6yTmSMmQZMA4iOjjZlNFeucGgt1A6FsPaAdVf+5cbDPL/4j7vyuy9rhb8OP1TK4zkT6IlAixLbzYGkEtshQGfgR/s8Hk2AxSIyXL8YrQYOroGIS8HPj7zCYp74fAtfbUoiumUDXr9Z78qV8ibOBHoMECUikcBhYBRw26mDxphM4HTnrIj8CDyuYV4NZB2B4/uh11iOZeVx76w4Nidk8NigdjwwsK3elSvlZcoMdGNMkYhMApZhDVucbozZLiJTgFhjzOLKLlJdoENW//muWl24e+qvZJws5H939GRw5yZuLkwpVRmcGodujFkCLCm179lztB1Q8bKUK+TGr8bPrzZDF2TRuF4wCyZcquPKlfJi+qSoF0rJzmfeb4e4duMqUk0bxl4Zxf9d1ZaQwJruLk0pVYk00L1Esc2wfn8a839LYOm2I9QuzmZS4CHC+zzG5UM7urs8pVQV0ED3cJsTMlgQl8B325JJPZFPSGAN7uzbivFNduH3rSG04wB3l6iUqiIa6B4qOSuPV5b+zpcbD1O7pj9XdWjEsK5NGdi+EbUD/GHpDPCreXoRZ6WU99NA9zD5RcV8tHo///0+nqJiw8SBbbh/QFvq1LJ/lMbAj6/A+v9Bx+F/rECklPJ6GugewhjD978fY8o3OziYdpJBnRrz9LCOtGxY4sGgkgs7dxsN1//HfQUrpaqcBroHyC8q5v/mbmT5jmTahAcz457e9G8XfmajrCMw/zZI2gjXPA+XP+xwYWellPfSQK/mjDE8sXALy3ck88TgDoy7IpKapWdDTNoE80ZDXiaMmgMdhrmnWKWUW2mgV3NvrdzDok1JPP6ndtw/wMECzju/gc/HWVPjjl0GTbpUfZFKqWpBA70a+3JjIv9ZtYebejRn4sC2ZzfITLTWBm3UEW77FOo0qvoilVLVhgZ6NRR/LJu3V8Xz9ZYk+rYO5eUbuyCO+sOXPQXGBjd/omGulNJAr04yTxby3OJtfLU5ido1/ZnQvw0PDGhDQA0HKwjt/QF2LIKBT0GDllVfrFKq2tFAryb2p+YwdkYMCeknue/KNoy/sjWhwQGOGxcVwJK/QoNIuOzBqi1UKVVtaaBXA2v2pnL/7A34CcwZ15fekaHnf8O6dyFtD9z22RmLPSulfJsGupv9uOsY42bEEhkWzEd39SKiYdD53xDzIayaAu2HQrtrq6ZIpZRH0EB3o9+PZjFp7kbaNQ5h/n19qXu+6W2Li+C7yRDzAURdCze8X3WFKqU8goNv284mIoNFZJeIxIvIZAfHJ4jIVhHZJCKrRaST60v1Lsey8rjn4xiCa/nz0d3R5w/z3AyYM9IK80snweh5EFi36opVSnmEMu/QRcQfmAoMwlowOkZEFhtjdpRoNtcY8z97++HAG8DgSqjXK5wsKGLczFgycgv57L5LaVrvPBNope2FubfC8QMw/L/Q484qq1Mp5Vmc6XLpDcQbY/YBiMh8YARwOtCNMVkl2gcDxpVFepOCIhv3z97AtsOZTLsz+uwl4bKT4WSq9Tp9Py+8TcIAAA5iSURBVHw1EcQP/vIVtLq86gtWSnkMZwK9GZBQYjsR6FO6kYhMBB4FAoCrXFKdl7HZDI8v2MxPu1N45cYuXNOp8ZkNEuPg48FQXPDHvvAOMHo+hEZWbbFKKY/jTKA7mrLvrDtwY8xUYKqI3AY8Ddx11olExgPjASIiIspXqYczxvDC19tZvDmJJwZ3YFTvUtdvK4Ylj0HtBjDkNWumRL8aEHkl1ApxT9FKKY/iTKAnAi1KbDcHks7Tfj7wnqMDxphpwDSA6Ohon+qW+eCXfcxYe5D7rmzteJKtDTOtqW9v/BAu/nPVF6iU8njOjHKJAaJEJFJEAoBRwOKSDUQkqsTmMGCP60r0fKv3pPLK0t8Z2qUJk4d0OLvByXRY9QK07AddRlZ9gUopr1DmHboxpkhEJgHLAH9gujFmu4hMAWKNMYuBSSJyDVAIHMdBd4uvSkg/yaR5G2jbqA6vj+z2xyRbRflgK7Jer3we8rJg6Ou6KIVS6oI59WCRMWYJsKTUvmdLvH7IxXV5hdyCYu6bFYfNZph2ZzTBp9b93PcTzBsFhSf/aNx3IjTW4ftKqQunT4pWEpvN8NiCTew8msX0u3rRKsy+9mdRPnz7qDXdbfQ91r5aIdYaoEopVQEa6JXk9eW7WLL1KE8N7cjADiXmKl87FdLi4faFEDXIfQUqpbyOU4/+q/L5LCaB937cy219Ihh3RYnx45mJ8PPr0OE6DXOllMtpoLvYmr2pPPnlVq6ICuOF4RefudLQqRWGrv2n+wpUSnkt7XJxoYT0k0ycs4HIsGCm3t6Dmgd+gkX3W6NZjLEe6R/4tK4wpJSqFBroLnJqREuRzTDtL/bZEzfOtkaydL7JalSnCVyuKwwppSqHBroLGGP4+xdb2Hk0i4/uiiYyLBiKCyF+hdVfft2b7i5RKeUDtA/dBT5avZ9Fm5J49Jp2XNXBPuFWwnrIy4R2OouwUqpqaKBX0PLtR/nHkp0M6dyEiQPb/nFg11LwD4A2A91XnFLKp2igV8DWxEwemr+Jrs3r88Ytl+DnV2JEy+7voFU/nSlRKVVlNNAvUFJGLmNnxBAaHMAHf+lJ7QD/Pw6mxlsPD2l3i1KqCmmgX4BdR7O55f215BYU8/GYXjQKCTyzwe7vrN810JVSVUgDvZxW7Uzmxnd/paDIxuxxfWjX2EGXyu7voFEnHW+ulKpSGuhOKiq28d/v9zBuZiyR4cF8NelyurWof3bD3Aw4uAbaXVv1RSqlfJqOQ3dC/LFsHluwhc0JGVzXtSmvjexKUMA5/tPFrwRTDO2GVG2RSimfp4FehjnrD/LC1zsIDvDnndHdua5r0zPnZylty6fWE6HNo6uuSKWUQgP9vNbEp/L0om1cGRXOv27uRnhIrfO/ISMB9qyAKx8HP//zt1VKKRdzqg9dRAaLyC4RiReRyQ6OPyoiO0Rki4isEhGP/zbwWHYeD87fRJvwOrx3R4+ywxxg4yzr9+53Vm5xSinlQJmBLiL+wFRgCNAJGC0ipddK2whEG2O6AguB11xdaFUqthkenr+JE/mFvHt7j3P3l5/xpiLYMAvaXq2jW5RSbuHMHXpvIN4Ys88YUwDMB0aUbGCM+cEYc2qBzHVAc9eWWXWMMfxr+S7W7E1jyvDOjoclOhK/ArKToOeYyi1QKaXOwZk+9GZAQontRKDPedqPBZY6OiAi44HxABEREU6WWHUSj59k8udbWR2fys09m3NzdDn+XYr92PoyVIcrKqXcxJlAdzSkwzhsKHIHEA30d3TcGDMNmAYQHR3t8BzuYIzh05gEXvp2J8YY/nFDZ27rHXH+0SwlZSRYd+j9HgX/mpVbrFJKnYMzgZ4ItCix3RxIKt1IRK4BngL6G2PyXVNe5cvOK+TvX2zlmy1HuKxNQ169qSstQoPKd5LYj6wViXrol6FKKfdxJtBjgCgRiQQOA6OA20o2EJHuwPvAYGPMMZdXWUm2J2Uycc4GEo7n8rfB7ZlwZZszZ0x0Rvo+WPsudBkJDVpVSp1KKeWMMgPdGFMkIpOAZYA/MN0Ys11EpgCxxpjFwOtAHWCBvZvikDFmeCXWXWGLNyfx1wWbqR9Uk3n39qV3ZOiFnWjpZKubZdCLri1QKaXKyakHi4wxS4AlpfY9W+L1NS6uq9LYbIa3Vu3h7VV76NWqAe/d0ZOwOk6MMXdk11LYswz+9BLUberaQpVSqpx86knRwmIbD3+6iW+3HOHmns156YbO1KpxgU90FubC0r9BeAfoM8G1hSql1AXwmUA3xvDMom18u+UIk4d04L4rWzs/iqU0mw2WPw0Zh+Cur3Vki1KqWvCZQH//533Mj0lg4sA2TOjf5sJPlH8CvrwPfv8G+j4AkVe6rkillKoAnwj0pVuP8MrS37mua1MeG9S+fG/OzYBjO6zXxQWw7Gk4th0Gv6JdLUqpasXrA31/ag6PfLaJ7hH1+dfN3co3LLEwF6YNgOP7/9hXqy7c9hlEDXJ5rUopVRFeH+j/+HYnNfz8eP+OngTWLOcXoKvftMJ8+DtQ3z5VQVh7HdGilKqWvDrQV+9JZeXOZJ4Y3IFGdQPLfkNJ6ftg9VvQ5Wbo8ZfKKVAppVzIa9cULSq28eI3O2gRWpsxl7cq35uNgaVP6ANDSimP4rWB/mlsAruSs3lySMfyd7XsWgp7lsOAydq9opTyGF4Z6InHT/LG8t30jgxlcOcm5XxzHHz9oD4wpJTyOF4V6MYY5q4/xOC3fiGvsJjnru9UvoeHti6ET4ZCzSC4ZaY+MKSU8ihe86XowbQcnl60jV/2pDo3DW5xEcROh6xEa/tECmyeCxGXwa2zIbhh1RSulFIu4vGBnpNfxNQf4vnwl/3U9Bde/HNnbu8dcf7x5nmZsGAM7F0F/rVABBDoeTcMeR1qBFRV+Uop5TIeG+gFRTY+35DIWyt3k5yVz43dm/HEkA40Lmt4Yvo+mDsK0vfC9W9Dz7uqpmCllKpkHhfop4L8v9/Hczgjl0ta1Ofd23vQs6UT85lnH4UPrwFjgzsXQeQVlV+wUkpVEY8L9LdX7eG/P8TTrUV9XrqhMwPahTv/xefyZyA/G+77BRp1qNxClVKqinlcoN/RtyU9WzUoX5ADHPgVtn4GV/5Nw1wp5ZWcGrYoIoNFZJeIxIvIZAfHrxSRDSJSJCIjXV/mH5rUC2Rg+0blC/PiQljyONSLgH6PVF5xSinlRmUGuoj4A1OBIUAnYLSIdCrV7BBwNzDX1QW6xG8fWFPgDn4ZAs4zlFEppTyYM10uvYF4Y8w+ABGZD4wAdpxqYIw5YD9mq4Qaz7RhFqz9b/nec/wAtL0GOgyrlJKUUqo6cCbQmwEJJbYTgT4X8sNEZDwwHiAiIuJCTgFBoRBezkUqmvWEgU/ax5srpZR3cibQHaWguZAfZoyZBkwDiI6OvqBz0GGY3mkrpZQDznwpmgi0KLHdHEiqnHKUUkpdKGcCPQaIEpFIEQkARgGLK7cspZRS5VVmoBtjioBJwDJgJ/CZMWa7iEwRkeEAItJLRBKBm4H3RWR7ZRatlFLqbE49WGSMWQIsKbXv2RKvY7C6YpRSSrmJV82HrpRSvkwDXSmlvIQGulJKeQkNdKWU8hJizIU931PhHyySAhy8wLeHAakuLMdT+OJ1++I1g29ety9eM5T/ulsaY8IdHXBboFeEiMQaY6LdXUdV88Xr9sVrBt+8bl+8ZnDtdWuXi1JKeQkNdKWU8hKeGujT3F2Am/jidfviNYNvXrcvXjO48Lo9sg9dKaXU2Tz1Dl0ppVQpHhfoZa1v6g1EpIWI/CAiO0Vku4g8ZN8fKiIrRGSP/fcG7q7V1UTEX0Q2isg39u1IEVlvv+ZP7TN+ehURqS8iC0Xkd/tnfqmPfNaP2P98bxOReSIS6G2ft4hMF5FjIrKtxD6Hn61Y3rZn2xYR6VHen+dRge7k+qbeoAh4zBjTEegLTLRf52RglTEmClhl3/Y2D2HN6nnKq8Cb9ms+Dox1S1WV6z/Ad8aYDkA3rOv36s9aRJoBDwLRxpjOgD/W1Nze9nl/Agwute9cn+0QIMr+azzwXnl/mEcFOiXWNzXGFACn1jf1KsaYI8aYDfbX2Vh/wZthXesMe7MZwJ/dU2HlEJHmwDDgQ/u2AFcBC+1NvPGa6wJXAh8BGGMKjDEZePlnbVcDqC0iNYAg4Ahe9nkbY34G0kvtPtdnOwKYaSzrgPoi0rQ8P8/TAt3R+qbN3FRLlRCRVkB3YD3Q2BhzBKzQBxq5r7JK8RbwN+DUYuMNgQz7nPzgnZ93ayAF+Nje1fShiATj5Z+1MeYw8C/gEFaQZwJxeP/nDef+bCucb54W6C5b39QTiEgd4HPgYWNMlrvrqUwich1wzBgTV3K3g6be9nnXAHoA7xljugM5eFn3iiP2fuMRQCRwERCM1eVQmrd93udT4T/vnhboPrO+qYjUxArzOcaYL+y7k0/9L5j992Puqq8SXA4MF5EDWF1pV2Hdsde3/y85eOfnnQgkGmPW27cXYgW8N3/WANcA+40xKcaYQuAL4DK8//OGc3+2Fc43Twt0n1jf1N53/BGw0xjzRolDi4G77K/vAr6q6toqizHm78aY5saYVlif6/fGmNuBH4CR9mZedc0AxpijQIKItLfvuhrYgRd/1naHgL4iEmT/837qur3687Y712e7GPiLfbRLXyDzVNeM04wxHvULGArsBvYCT7m7nkq6xn5Y/6u1Bdhk/zUUq095FbDH/nuou2utpOsfAHxjf90a+A2IBxYAtdxdXyVc7yVArP3zXgQ08IXPGngB+B3YBswCannb5w3Mw/qOoBDrDnzsuT5brC6XqfZs24o1AqhcP0+fFFVKKS/haV0uSimlzkEDXSmlvIQGulJKeQkNdKWU8hIa6Eop5SU00JVSyktooCullJfQQFdKKS/x/4Ypypwtn3YZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores)\n",
    "plt.plot(test_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU + Sigmoid + Cross Entropy + L1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit2(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        \n",
    "        L1_Loss = 0\n",
    "        for param in net.parameters():\n",
    "            L1_Loss += t.sum(t.abs(param))\n",
    "    \n",
    "        loss = criterion(outputs, labels.long())\n",
    "        loss += 0.001 * L1_Loss\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net2 = Net()\n",
    "criterion2 = nn.CrossEntropyLoss()\n",
    "optimizer2 = optim.SGD(net2.parameters(), lr = 1e-1, weight_decay=1e-2)\n",
    "loss_scores2, train_scores2, test_scores2 = fit2(net2, criterion2, optimizer2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3hU1fr28e+TBEIJvVcpgoBKHTohHIoCUkSxwFFBRcACCPj601M8Ht9z9GehN5Ui1iOIFWlSlIQqCdJBCCiCtIh06az3jwznxRgggQk7M3N/rmuuzJ69mHm2G2921qy9ljnnEBGR4BfhdQEiIhIYCnQRkRChQBcRCREKdBGREKFAFxEJEVFefXDRokVdhQoVvPp4EZGglJSU9Itzrlh6+zwL9AoVKpCYmOjVx4uIBCUz236xfepyEREJEQp0EZEQoUAXEQkRCnQRkRChQBcRCREKdBGREKFAFxEJEUEX6PuPnuSF6Rs4cuK016WIiGQrQRfoi7fuZ/KSH7h1WDzxm1O8LkdEJNsIukDvVKs0Hz/ahNw5I3lg0rc8+8kaXa2LiBCEgQ5Qp3whZvSPpU9cJaas2KGrdRERMhjoZlbQzKaZ2SYz22hmjdPsb2Fmh8xslf/xXNaU+//lyhHJs+2q/+5q/ZmP13BYV+siEqYyOjnXCGC2c66rmeUE8qTTJsE51yFwpWXM+av14fO28Gb8VhZuTuHFO27mTzcUv9aliIh46rJX6GaWH2gOTARwzp1yzh3M6sIyI1eOSJ5pV41PHmtKvlxRPPjWCgZNXcXB3055XZqIyDWTkS6XSkAK8JaZfWdmE8wsbzrtGpvZajObZWY3pvdGZtbbzBLNLDElJfB93rXLFWR6v2b0a3k9n6/aReuh8cxetyfgnyMikh1lJNCjgLrAOOdcHeAY8EyaNiuB65xztYBRwGfpvZFz7k3nnM855ytWLN352a9adFQkg2+5gc8fb0rxfNH0fS+Jxz9YyS9HT2bJ54mIZBcZCfSdwE7n3HL/9jRSA/6/nHOHnXNH/c9nAjnMrGhAK82km8oU4PMnmvLULVWZu34vbYYu5LPvfsY552VZIiJZ5rKB7pzbA+wwsxv8L7UCNlzYxsxKmpn5nzfwv+/+ANeaaTkiI3iiZRVm9G9GhaJ5eXLKKh6avIJdB497XZqISMBldBx6P+B9M1sD1AZeNLO+ZtbXv78rsM7MVgMjgXtdNroUrlIiH9P6NuHvHWqwbNuv3DIsnveWbefcuWxToojIVTOvctfn8zkv1hT9af9vPPvpGhYn76dBxcL87x03U6lYzDWvQ0TkSphZknPOl96+oLxT9GqUL5KH9x5uyCtda7Jp92Hajkhg7DfJnD57zuvSRESuStgFOoCZcbevHPMGxdGqWnFemf09nUcvZu3OQ16XJiJyxcIy0M8rnj8X4+6rx+v31eOXoyfpPGYRL87cyPFTZ70uTUQk08I60M9re1NJ5g6K45765Xgzfhu3Do9n0ZZfvC5LRCRTFOh+BXLn4KU7avKfRxoRGWHcN3E5g6eu5sAxTR8gIsFBgZ5G48pFmDUglsdaVObzVT/TeuhCPl+lG5JEJPtToKcjV45Inm5bjen9mlG2UG4GfLiKByevYMevv3ldmojIRSnQL6F6qfx88lhTnutQg29/SL0haULCNs5oiKOIZEMK9MuIjDAealaRuYPiaFy5CP+asZEuY5ew7mcNcRSR7EWBnkFlCuZmYg8fo7vXYfehE3Qes5h/z9jAb6fOeF2aiAigQM8UM6NDzdLMHxTH3b6yjE/4gTZD4/n6+31elyYiokC/EgXypA5xnNqnMblzRvLgWyt44oOV7DtywuvSRCSMKdCvQoOKhZnRvxkDW1flq/V7aT1kIR8s/0mzOIqIJxToVyk6KpIBrasw68lYapTOz18+Xcvdbyxl894jXpcmImFGgR4glYvF8J9HGvFq15psTTlK+xEJvDpnEydOa14YEbk2FOgBZGbc5Z/FsXPtMoz5eiu3Do8nYUvgF8QWEUlLgZ4FisREM+TuWnzQqyERZtw/8VsGfPgdKUe0ULWIZB0FehZqcn1RZg2IpX+rKsxau4dWQ77Rl6YikmUU6FksV45IBrWpyswBsVQvlfql6V1vLGXTnsNelyYiIUaBfo1cXzyGD3s34rW7arEt5SgdRi7ipVkbdaepiASMAv0aMjO61ivLgsEtuKNuGd5YuI02Q+OZv3Gv16WJSAhQoHugUN6cvNK1FlP7NCZPzkgefjuRPu8msuvgca9LE5EglqFAN7OCZjbNzDaZ2UYza5xmv5nZSDNLNrM1ZlY3a8oNLal3msbydNsbWLg5hTZDF2p6XhG5Yhm9Qh8BzHbOVQNqARvT7G8HVPE/egPjAlZhiMsZFcFjLa5n7sA4GlQszL9mbKTj6MWs/OmA16WJSJC5bKCbWX6gOTARwDl3yjl3ME2zzsA7LtUyoKCZlQp4tSGsXOE8TOpZn3F/rsuBY6e4c9wSnv1kLYd+O+11aSISJDJyhV4JSAHeMrPvzGyCmeVN06YMsOOC7Z3+137HzHqbWaKZJaak6O7JtMyMdjeXYt7gOB5uWpGpiTtoOeQbPk7aqTVNReSyMhLoUUBdYJxzrg5wDHgmTRtL58/9IYGcc28653zOOV+xYsUyXWy4iImO4m8dajD9iWaUL5KHwR+tptv4ZSTv04RfInJxGQn0ncBO59xy//Y0UgM+bZtyF2yXBXZdfXnhrUbp/HzctwkvdrmZjbuP0G5EAq/M3sTxU5rwS0T+6LKB7pzbA+wwsxv8L7UCNqRp9gXwgH+0SyPgkHNud2BLDU8REUb3huWZPziOTrXKMPabrbQeupB5GzR2XUR+L6OjXPoB75vZGqA28KKZ9TWzvv79M4FtQDIwHngs4JWGuaL+Cb+m9G5E3uhIer2TyCPvJPKzxq6LiJ959WWbz+dziYmJnnx2sDt99hwTF/3AiHlbAOjfqgoPN6tIzijdJyYS6swsyTnnS2+fEiAI5YiMoG9cZeYOak6zKkV5efYmbhuZwLJt+70uTUQ8pEAPYmUL5WH8Az4m9vBx/PRZ7n1zGQOnrNK86yJhSoEeAlpVL8HcgXE88afr+XLNLloO+YZ3lv7IWc27LhJWFOghInfOSJ669QZmP9mcmmUL8Nzn67l9zGJW7Uh7U6+IhCoFeoipXCyG9x5uyMhuddh7+ARdxi7mL5+u5eBvp7wuTUSymAI9BJkZnWqVZv7gOB5qWpEpK3bQcshCpibu0PJ3IiFMgR7C8uXKwd871ODLfs2oVDQvT09bw11vLGXDLi1/JxKKFOhhoHqp/Ezt05hXu9bkx1+O0WFUAv+cvp7DJzSTo0goUaCHiYgI4y5fORYMbkH3huWZvORHWg1ZyGff/ayZHEVChAI9zBTIk4N/3X4znz/elNIFcvHklFV0G7+MLXs1k6NIsFOgh6maZQvyyWNN+XeXm/47k+NLMzdy7OQZr0sTkSukQA9jkRHGnxtex4LBcdxRtwxvxG+j1ZCFzFizW90wIkFIgS4UiYnmla61+PjRxhTKm5PHP1jJA5O+ZVvKUa9LE5FMUKDLf9W7rjDTn2jK8x1rsOqng7QdnsCrc7SghkiwUKDL70RFRtCzaUXmPxVHh5qlGPN16oIac9bvUTeMSDanQJd0Fc+Xi6H31GZK70bEREfR590kHpq8gu37j3ldmohchAJdLqlhpSJ82b8Zf7utOit+PECbYfEMnbuZE6fVDSOS3SjQ5bJyREbQK7YS8wfH0fbGkoycv0XrmopkQwp0ybAS+XMxslsd/vNII3LnSF3XVN0wItmHAl0yrXHlIswcEMtf21dn+bb9tBkWzzB1w4h4ToEuVyRHZASPNK/E/MEtuPXGkoyYv4U2wxYyf6O6YUS8okCXq1KyQC5GdavDB70aEh0VycNvJ/Lw5BX8tP83r0sTCTsZCnQz+9HM1prZKjNLTGd/CzM75N+/ysyeC3ypkp01ub4oM/vH8pf21Vi6bT+thy1k+Dx1w4hcS1GZaPsn59wvl9if4JzrcLUFSfDKGRVB7+aV6VirNP+esZHh87bwycqfeb5TDVpWK+F1eSIhT10uEnClCuRmdPe6vN+rITmjInhociK93l7Bjl/VDSOSlTIa6A74ysySzKz3Rdo0NrPVZjbLzG4MUH0SxJr6u2GebVeNJVv303qoumFEspJlZH4OMyvtnNtlZsWBuUA/51z8BfvzA+ecc0fNrD0wwjlXJZ336Q30Bihfvny97du3B+o4JJvbfeg4/56xkS/X7KZ84TzqhhG5QmaW5JzzpbsvsxMumdnzwFHn3GuXaPMj4LtUn7vP53OJiX/4flVC3JLkX3jui/Uk7ztK6+rF+UfHGylXOI/XZYkEjUsF+mW7XMwsr5nlO/8cuAVYl6ZNSTMz//MG/vfdf7WFS+hpom4YkSyTkT70EsAiM1sNfAvMcM7NNrO+ZtbX36YrsM7fZiRwr9Ncq3IROaMi6BNXmQWDW3DLjSUZPm8LtwyL101JIlcp010ugaIuFzkvbTfMcx1upHwRdcOIpOequlxEstqFNyUt2aqbkkSulAJdsoXzNyUt8M8No24YkcxToEu2cuHcMDmjInj47dSbkjQ3jMjlKdAlW/rDaBh1w4hclgJdsq3zo2HmD47jlhol/tsNs2CTumFE0qNAl2zv/NwwH2huGJFLUqBL0EjvpqSR87eoG0bET4EuQeXCbpjWNUowdO5mbh0ez9ff7/O6NBHPKdAlKJUqkJsx3evy3sMNiYwwHnxrBb3fSWTnAXXDSPhSoEtQa1alKLMHNOd/2lYjYcsvtB66kNELtnDyjLphJPwo0CXo5YyK4NEWlZk3OI4WVYvz2lebaTc8gYQtKV6XJnJNKdAlZJQpmJvX76/H5Afrc8457p/4LY+9n8TuQ8e9Lk3kmlCgS8hpcUNx5gxszuA2VZm/cR+thizkjYVbOXXmnNeliWQpBbqEpOioSPq1qsK8QXE0qVyEl2Zt4raRCSzdqmn6JXQp0CWklSuchwk96jPhAR/HT5+l2/hlPPnhd+w7fMLr0kQCToEuYaF1jRLMHRhHv5bXM3PtHloNWcjkxT9w5qy6YSR0KNAlbOTOGcngW25gzsDm1C5fkOenb6DzmMWs/OmA16WJBIQCXcJOxaJ5eeehBozpXpf9R09xx9glPPvJGg4cO+V1aSJXRYEuYcnMuK1mKeYNjuOR2IpMTdxJyyHfMHXFDs6d03K4EpwU6BLWYqKj+OttNZjRvxmVi8Xw9MdruPuNpWzac9jr0kQyTYEuAlQrmZ+pfRrzSteabE05ym0jF/HvGRs4dvKM16WJZJgCXcQvIsK421eOBYNbcLevLOMTfqD10IXMWrsb59QNI9mfAl0kjUJ5c/LSHTX5+NEmFMyTk0ffX8mDk7WuqWR/GQp0M/vRzNaa2SozS0xnv5nZSDNLNrM1ZlY38KWKXFv1rivE9Cea8rfbqrPih19pM2whY75O1hQCkm1l5gr9T8652s45Xzr72gFV/I/ewLhAFCfitajICHrFVmLe4DhaVivOq3O+p/3IBJZt0xQCkv0EqsulM/COS7UMKGhmpQL03iKeK1UgN+Puq8dbPetz4vRZ7n1zGU99tJpfNXZdspGMBroDvjKzJDPrnc7+MsCOC7Z3+l/7HTPrbWaJZpaYkqK5qiX4/KlaceYOjOOxFpX57Luf/zt2XV+aSnaQ0UBv6pyrS2rXyuNm1jzNfkvnz/zhb7hz7k3nnM855ytWrFgmSxXJHnLnjOTpttWYOSCWKsVTx67f88Yytuw94nVpEuYyFOjOuV3+n/uAT4EGaZrsBMpdsF0W2BWIAkWyq6ol8jGld2NevvNmvt97hPYjE3htzvecOK3l78Qblw10M8trZvnOPwduAdalafYF8IB/tEsj4JBzbnfAqxXJZiIijHvql2fB4Dg61izN6K+TaTs8nkVbfvG6NAlDGblCLwEsMrPVwLfADOfcbDPra2Z9/W1mAtuAZGA88FiWVCuSTRWJiWboPbV5v1dDAO6buJyBU1ax/+hJjyuTcGJefZnj8/lcYuIfhrSLBL0Tp88y5utkXl+4lbzRUfylfXXuqlcWs/S+ahLJHDNLusjwcd0pKhJouXKkzrs+s38s1xeL4elpa+g2fhnbUo56XZqEOAW6SBapUiIfU/s05sUuN7N+12Hajkhg1PwtutNUsowCXSQLRUQY3RuWZ/7gOG6pUYIhczfTYVQCSdu1SpIEngJd5Booni8Xo7vXZVJPH0dPnKHr60t47vN1HDlx2uvSJIQo0EWuoZbVSvDVoDh6NK7Au8u202ZoPPM27PW6LAkRCnSRaywmOornO93IJ482oUDuHPR6J5HHP1hJyhENcZSro0AX8Uid8oWY3q8Zg9tUZe76vbQeupCPEjUvjFw5BbqIh3JGRdCvVRVmPRlL1RIx/J9pa3hg0rfs+FWLaUjmKdBFsoHKxWKY0rsx/7fzjazcfoBbhsUzadEPnD2nq3XJOAW6SDYREWHc37gCcwfF0ahSYV74cgN3vb6E5H2axVEyRoEuks2ULpibST3rM/ye2vzwyzHaj1jEmK+TOX1WNyTJpSnQRbIhM+P2OmWYOyiONjeW4NU533P7mMWs33XI69IkG1Ogi2RjRWOiGdO9Lq/fV4+9h0/SefRihn71vaYPkHQp0EWCQNubSjJvUHM61S7NyAXJdBy1iNU7DnpdlmQzCnSRIFEwT06G3l2bST19HDp+mjvGLeGV2Zs4eUYrJEkqBbpIkEmdPqA5d9Ytw9hvttJx1CLW7NTVuijQRYJS/lw5eKVrLd7qWZ/Dx8/QZewSXpvzva7Ww5wCXSSI/alaceYMbE6XOmUY/XUynUYtZt3PGgkTrhToIkGuQO4cvHZXLSb28HHgt1PcPmYxw+Zu1rj1MKRAFwkRraqX4KuBzelQsxQj5m+hy9jFfL9Hd5mGEwW6SAgpmCcnw++tw+v31WP3wRN0HLWINxZu1ZwwYUKBLhKC2t5Ukq8GNqdlteK8NGsT9765lO37j3ldlmSxDAe6mUWa2Xdm9mU6+3qaWYqZrfI/egW2TBHJrCIx0Yy7ry7D7qnFpj1HaDcigfeXb9d86yEsM1foA4CNl9g/xTlX2/+YcJV1iUgAmBld6pRlzpPNqVu+EH/9dB0PTV7BvsMnvC5NskCGAt3MygK3AQpqkSBUumBu3nmoAc93rMGSrfu5dXg8s9bu9rosCbCMXqEPB54GLjUO6k4zW2Nm08ysXHoNzKy3mSWaWWJKSkpmaxWRqxARYfRsWpEZ/WMpVzgPj76/kkFTV3H4xGmvS5MAuWygm1kHYJ9zLukSzaYDFZxzNYF5wNvpNXLOvemc8znnfMWKFbuigkXk6lxfPIaPH21C/1ZV+HzVLtoNT2D5tv1elyUBkJEr9KZAJzP7EfgQaGlm713YwDm33zl3fsny8UC9gFYpIgGVIzKCQW2q8lHfxuSINO4dv4yXZ2/StLxB7rKB7px71jlX1jlXAbgXWOCcu+/CNmZW6oLNTlz6y1MRySbqli/EjP6x3Fu/HOO+2UqXsYu15F0Qu+Jx6Gb2gpl18m/2N7P1ZrYa6A/0DERxIpL18kZH8dIdNXnz/nrsPnSCDqMW8e4yDW8MRubVSfP5fC4xMdGTzxaR9O07fIKnpq0hfnMKraoV5+WuNSkaE+11WXIBM0tyzvnS26c7RUXkv4rnz8XknvX5R8caJCT/QtvhCSzcrBFpwUKBLiK/ExFhPNi0Il880ZTCeXPQY9K3vDB9AydOa6717E6BLiLpqlYyP1880Yweja9j0uIfuH3MYrbs1Rem2ZkCXUQuKleOSP7Z+SYm9fSRcuQkHUcv0nww2ZgCXUQuq2W1EswaEEv9CoX566fr6PteEgd/O+V1WZKGAl1EMqR4/ly8/WAD/tq+Ogs27aPdCN1hmt0o0EUkwyIijEeaV+LjR5sQHRVBt/HLGDZ3M2e03F22oEAXkUyrWbYgX/aP5fY6ZRgxfwvdxy9n18HjXpcV9hToInJFYqKjGHp3bYbdU4v1uw7RbkQCc9bv8bqssKZAF5Gr0qVOWWb0j6V84Tz0eTeJf3y+TmPWPaJAF5GrVqFoXj5+tAm9mlXk7aXb6TJ2CVtTjnpdVthRoItIQOSMiuBvHWowqaePPYeO03HUIqYl7fS6rLCiQBeRgEods96cm8sU4KmPVjNoyiqOnTzjdVlhQYEuIgFXskAuPnikEQNaVeGzVT/TcdQi1u865HVZIU+BLiJZIjLCGNimKu/3asTRk2foMnYJ7y79UdMGZCEFuohkqcaVizBrQCxNKhfh75+v59H3VnLouBamzgoKdBHJckViopnUoz7PtqvGvI17uW1kAt/9dMDrskKOAl1EromICKNPXGWm9m2Mc3DX60t5Y+FWzp1TF0ygKNBF5JqqW74QM/vH0rp6CV6atYmH317Br8c0c2MgKNBF5JorkCcH4+6rywudb2Rx8n7ajYjXzI0BoEAXEU+YGQ80rsAnjzUhT84ouo1fxsj5WzirLpgrpkAXEU/dVKYA0/s1o1Ot0gydu5n7Jy5n3+ETXpcVlDIc6GYWaWbfmdmX6eyLNrMpZpZsZsvNrEIgixSR0BYTHcWwe2rzStearPzpAO1GJLBwc4rXZQWdzFyhDwA2XmTfw8AB59z1wDDg5astTETCi5lxt68c059oRtGYaHpM+paXZ2/itBbPyLAMBbqZlQVuAyZcpEln4G3/82lAKzOzqy9PRMJNlRL5+OzxpnRrUI5x32zlnjeWsvPAb16XFRQyeoU+HHgauNg/lWWAHQDOuTPAIaBI2kZm1tvMEs0sMSVFv06JSPpy54zkpTtqMqpbHTbvPUr7EQnMXqfFMy7nsoFuZh2Afc65pEs1S+e1P3xV7Zx70znnc875ihUrlokyRSQcdaxVmhn9m1GhaF76vpfEc1o845IycoXeFOhkZj8CHwItzey9NG12AuUAzCwKKAD8GsA6RSRMXVckL9P6pi6e8c7S7dw+ZjHJ+7R4RnouG+jOuWedc2WdcxWAe4EFzrn70jT7Aujhf97V30aDSUUkIC5cPGPfkZN0HLWIqYk7NHNjGlc8Dt3MXjCzTv7NiUARM0sGBgHPBKI4EZELtaxWgpn9Y6lVrgBPT1vDgA9XcfiEZm48z7z6F87n87nExERPPltEgtvZc46xXyczfP4WShfMxahudaldrqDXZV0TZpbknPOlt093iopI0ImMMPq1qsLUPo04dw66jlvC2G+Sw37mRgW6iAStetcVZuaAWG69sSSvzP6e+yctZ28YTxugQBeRoFYgdw5Gd6/D/95xMyu3H6Tt8HjmbdjrdVmeUKCLSNAzM+5tUJ7p/ZpRskBuer2TGJZj1hXoIhIyri8ew2ePN+Fh/5j1TqMXsXH3Ya/LumYU6CISUqKjIvl7hxq8/VADfj12ms5jFjNp0Q9h8YWpAl1EQlJc1WLMfjKW5lWK8sKXG+jx1rchP8+6Al1EQlbRmGjGP+DjX7ffxIoff+XW4fEhPcmXAl1EQpqZcV+j6/iyXzPKFMpN3/eS+J9pazh28ozXpQWcAl1EwsL1xfPxyaNNeaxFZaYm7aD9yASSth/wuqyAUqCLSNjIGRXB022rMaV3Y86cddz1+hKGfPV9yKyKpEAXkbDToGJhZj8ZS5c6ZRm1IJk7xi4hed8Rr8u6agp0EQlL+XLlYMjdtXj9vrrsPPAb7UcuYkLCtqAe3qhAF5Gw1vamUswZ2JzmVYryrxkb6T5hGTt+Dc41TBXoIhL2iufLxfgHfLxyZ03W/XyYtsPj+fDbn4JuAQ0FuogIqcMb765fjtlPxlKzbEGe+WQtD01ewZ5DwXMzkgJdROQCZQvl4f1eDXm+Yw2WbtvPLcMW8nHSzqC4Wlegi4ikERFh9GxakdkDmlO1RD4Gf7SaR95JzPZzrSvQRUQuokLRvEzp05i/3VadhC2/0GboQqZl46t1BbqIyCVERhi9Yisxa0AsVUvk46mPVvPQ5BXsPnTc69L+QIEuIpIBlYrFMKVPY57r4O9bHxrPB8uz10gYBbqISAZFRhgPNavInCebc1OZAvzl07V0H7+c7fuPeV0aoEAXEcm064rk5f1eDXmxy82s/fkQtw6PZ3z8Ns56fJfpZQPdzHKZ2bdmttrM1pvZP9Np09PMUsxslf/RK2vKFRHJHiIijO4NyzN3UHOaVi7Kv2du5I6xi9mwy7sl7zJyhX4SaOmcqwXUBtqaWaN02k1xztX2PyYEtEoRkWyqVIHcTOjhY1S3Ouw8cJxOoxfx8uxNnixQfdlAd6mO+jdz+B/Z51sAERGPmRkda5Vm/uA4utQpw7hvttJ2eDyLk3+5pnVkqA/dzCLNbBWwD5jrnFueTrM7zWyNmU0zs3IXeZ/eZpZoZokpKSlXUbaISPZTME9OXr2rFh/0agjAnycsZ9DUVew/evKafL5lZsiNmRUEPgX6OefWXfB6EeCoc+6kmfUF7nbOtbzUe/l8PpeYmHiFZYuIZG8nTp9l9IJk3ojfSkx0FM+2r85d9cpiZlf1vmaW5JzzpbcvU6NcnHMHgW+Atmle3++cO/9P0Hig3hXUKSISMnLliOSpW29gRv9YKheL4elpa7jnjWVs2Zt1C2lkZJRLMf+VOWaWG2gNbErTptQFm52AjYEsUkQkWFUtkY+pfRrz8p03s3nfEdqNSGBCwrYs+ayoDLQpBbxtZpGk/gMw1Tn3pZm9ACQ6574A+ptZJ+AM8CvQM0uqFREJQhERxj31y9O6eglenLmJCkXyZsnnZKoPPZDUhy4iknkB60MXEZHsS4EuIhIiFOgiIiFCgS4iEiIU6CIiIUKBLiISIhToIiIhQoEuIhIiPLuxyMxSgO1X+MeLAtd2XsrsIRyPOxyPGcLzuMPxmCHzx32dc65Yejs8C/SrYWaJF7tTKpSF43GH4zFDeB53OB4zBPa41eUiIhIiFOgiIiEiWAP9Ta8L8Eg4Hnc4HjOE53GH4zFDAI87KPvQRUTkj4L1Cl1ERNJQoIuIhIigC3Qza2tm35tZspk943U9WcHMypnZ12a20czWm9kA/+uFzWyumTbz7/QAAANlSURBVG3x/yzkda1Zwcwizew7M/vSv13RzJb7j3uKmeX0usZAMrOCZjbNzDb5z3njcDjXZjbQ//d7nZn9x8xyheK5NrNJZrbPzNZd8Fq659dSjfTn2xozq5uZzwqqQPcvgzcGaAfUALqZWQ1vq8oSZ4DBzrnqQCPgcf9xPgPMd85VAeb7t0PRAH6/Lu3LwDD/cR8AHvakqqwzApjtnKsG1CL12EP6XJtZGaA/4HPO3QREAvcSmud6MtA2zWsXO7/tgCr+R29gXGY+KKgCHWgAJDvntjnnTgEfAp09ringnHO7nXMr/c+PkPo/eBlSj/Vtf7O3gdu9qTDrmFlZ4DZggn/bgJbANH+TkDpuM8sPNAcmAjjnTjnnDhIG55rUNY1zm1kUkAfYTQiea+dcPKlrLV/oYue3M/COS7UMKGhmpTL6WcEW6GWAHRds7/S/FrLMrAJQB1gOlHDO7YbU0AeKe1dZlhkOPA2c828XAQ465874t0PtnFcCUoC3/N1ME8wsLyF+rp1zPwOvAT+RGuSHgCRC+1xf6GLn96oyLtgC3dJ5LWTHXZpZDPAx8KRz7rDX9WQ1M+sA7HPOJV34cjpNQ+mcRwF1gXHOuTrAMUKseyU9/j7jzkBFoDSQl9TuhrRC6VxnxFX9fQ+2QN8JlLtguyywy6NaspSZ5SA1zN93zn3if3nv+V+//D/3eVVfFmkKdDKzH0ntTmtJ6hV7Qf+v5RB653wnsNM5t9y/PY3UgA/1c90a+ME5l+KcOw18AjQhtM/1hS52fq8q44It0FcAVfzfhOck9UuULzyuKeD8/cYTgY3OuaEX7PoC6OF/3gP4/FrXlpWcc88658o65yqQem4XOOf+DHwNdPU3C6njds7tAXaY2Q3+l1oBGwjxc01qV0sjM8vj//t+/rhD9lyncbHz+wXwgH+0SyPg0PmumQxxzgXVA2gPbAa2An/1up4sOsZmpP6atQZY5X+0J7U/eT6wxf+zsNe1ZuF/gxbAl/7nlYBvgWTgIyDa6/oCfKy1gUT/+f4MKBQO5xr4J7AJWAe8C0SH4rkG/kPq9wSnSb0Cf/hi55fULpcx/nxbS+oooAx/lm79FxEJEcHW5SIiIhehQBcRCREKdBGREKFAFxEJEQp0EZEQoUAXEQkRCnQRkRDx/wDcpaSCixFo8AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xUVf7/8dcnjZCQAAmEElqAgIQuAcFGEQUBwZ8iTda6i+7K7tpX3bWh31XXiooFEbHCCqgUg4hgAZQSWFpCSQgthBISCAFSZ87vjztoDAkZkplMZvJ5Ph48yC2593O94e3NmXPPEWMMSimlvJ+fpwtQSinlGhroSinlIzTQlVLKR2igK6WUj9BAV0opHxHgqRM3atTItGnTxlOnV0opr7Rhw4ZjxpjGZW3zWKC3adOGxMRET51eKaW8kojsK2+bNrkopZSP0EBXSikfoYGulFI+QgNdKaV8hAa6Ukr5CA10pZTyERroSinlIzzWD10ppdzKbodNn8CJA645XsM20H08+JXzHHwqEzZ+CMUFFR+r41CI7uWaukrQQFdK+Z78k/DFJNi1xLFCqnhAx7wRu76B69+GOvV+vzljE8y5GU6mO3eusKYa6EopVaGs3TB7PGSlwrUvQp8/gVQx0I2BX6bBssdhZhqM+xQatLa2bZsPCyZDSCRM+hGa96j6NVSSU4EuIkOBqYA/MMMY83yp7a2AD4EGjn0eMcYkuLhWpZQ6v9TlMO92ED/4w5fQtr9rjisCl06GqItg3h0wtfvvt7fqB2M+gnpRrjlfJVUY6CLiD0wDrgbSgfUistAYk1xit38Bnxtj3haROCABaOOGepVS6lzGwJq34Nt/QeOLYNxnEBHj+vO0HwyTfoCt88HYrHUhkXDxrRAQ5PrzXSBnntD7AKnGmDQAEZkDjAJKBroBwh1f1wcyXFmkUkqVy26zmjw2fwadroPr3zm3jduVItpC/4fcd/wqcKbbYjRQ8mPidMe6kp4CJopIOtbT+V/LOpCITBKRRBFJzMzMrES5SilVyvr3rTC/8mG46SP3hnkVFBTbWLg5g7Hv/sLq1GNuOYczT+hlfZpgSi2PB2YZY14WkX7AxyLSxRhj/903GTMdmA4QHx9f+hhKKXVhcg/Dimeg7UAY+FjVP/x0oYJiG3uPnSHlaC6bD5zgi40HyTpdSKuIEE4XFLvlnM4EejrQssRyC85tUrkTGApgjPlFRIKBRsBRVxSplFJlWvqY1e97+Ms1IsyzThWwLPkIS7Yd5ufdxyiyWc+t/n7CVRdFcXPf1lzRvhF+fu6p1ZlAXw/EikgMcBAYB0wotc9+4Cpgloh0AoIBbVNRSrnP7hVWl8H+j0Bku2o77cn8IpIzTmIcbQw5eUWs35vNuj3ZJGXkYDfQMqIut/ZrQ9cW9YmNCqNt41CCA/3dXluFgW6MKRaRycBSrC6JM40xSSIyBUg0xiwEHgDeE5H7sJpjbjPGaJOKUso9ivLh6wetDygvv8/tpzt+upBlyUdI2HaI1am/PXmfFRTgR8+WDZg8KJZr4prQuXk44oHfGJzqh+7oU55Qat0TJb5OBi5zbWlKKVWO1VMhe7fV1zww2OWHL7LZ2Zd1hjVpWSzZdog1adnY7IaWEXW5/bIYLmvfiCB/q09JcKAfnZqFV8sTeEX0TVGllHfJ2g0rX4bON0C7QS477I7DJ/ls7X5+2Z3F3qzTvz6Ft20Uyt3923Jtl2Yee/J2lga6Usp7GAMJD0JAHRjy7yofLr/IxpJth/hkzX427DtOUIAfl7dvxOC4JsRG1aNrdH3aR9Wr0SFekga6Usp7JH1pfRh67X8gvFmlD7Pn2Gk+W7uPuRvSOXGmiJhGofxreCduvLgFDUM9/8ZnZWmgK6Vqrvwc2PAhFOVZy4kzoVl36P3HCz5Ukc3Od8lH+HTtflalHiPAT7imcxNuvqQ1/dpGuq0rYXXSQFdK1VxfPwhbP/9tuW5DGPEa+JX/AaQxhpP5xWSfLuTYqQI2HzjBmrRs1u/NJieviOgGdXnwmg6MiW9JVLjrP1D1JA10pVTNlPajFeZXPgQDHrPWiZT7ApExhi82HuTfCdvJOl34u21tIkMY2rkpV8c1YeBFUfj7wNN4WTTQlVI1T3EBfH2/NUvQFQ+UP0uQw8ETeTz2xVZ+3JVJr9YN+fOAdkTWC6JhSBCdmoXTxMeexMujga6UqnlWv25NUHHzfAis+7tNdrvh2+TDzF53gMM5+WSdLiT7dAHBgf48dV0ct/Rr4xPt4ZWhga6Uqlmy98DKlyDueogd/Otqu92weOsh3lyRwq4jp2gZUZdOTcO5uHUDGtWrw5j4lrSMCPFg4Z6nga6UqjnO9jP3C4Chz/26+tipAh6cu5kfdmYSG1WPqeN6MKJbc59tC68sDXSlVM2RvABSv4Mhz0F4cwB+Tj3Gvf/dxIm8Ip4e2Zk/9G1da5tUKqKBrpSqGQpy4ZtHoWlX6DOJY6cKeGXZLmav20/bRqHMur0Pcc3DKz5OLaaBrpSqGb5/DnIPUXDjLD5cvY83lqeSV2Tjtkvb8NCQjoQEaVxVRP8LKaU840gS7EywBty2FWDWvsPO6Bu59bNcjpw8xqCLonhsWCfaR9XMKeVqIg10pVT12zoPFtwDxfm/rtpHc8bsvoZOMaG8OqYHl7Zv5MECvZMGulKq+thtsHwKrH6N3CZ9mBL8EAtSCrDZDVfERjFjUAf6xER4ukqv5VSgi8hQYCrWjEUzjDHPl9r+KjDQsRgCRBljGriyUKWUD/j6ftgwi1X1R3L7vtGEhQp3XBHL+D4taR0Z6unqvF6FgS4i/sA04GqsCaPXi8hCxyxFABhj7iux/1+Bnm6oVSnlxYp2ryRwwyxm2ofzQtYE/jyoLXf1b0doHW0ocBVn/kv2AVKNMWkAIjIHGAUkl7P/eOBJ15SnlPIFW/Yepf6nf8bP3pgtHSazYkRPohvUrfgb1QVxJtCjgQMlltOBS8raUURaAzHAinK2TwImAbRq1eqCClVKeZ+8QhuvfrcL/59f4x8BB9h05bu8NvhST5fls5wJ9LJeyTJlrAMYB8wzxtjK2miMmQ5MB4iPjy/vGEopH7AmLYtH5m+hOHsfK4K/pKj9MHoMHufpsnza+cektKQDLUsstwAyytl3HDC7qkUppbyX3W54dnEy46avIdx+giXRswgKCCBw+H88XZrPcybQ1wOxIhIjIkFYob2w9E4i0hFoCPzi2hKVUt7CZjf8Y/4WZqzaw0Pd8vkq8F+EndgO178FDVpWfABVJRU2uRhjikVkMrAUq9viTGNMkohMARKNMWfDfTwwxxijTSlK1UI2u+GhuZv54n8HebP7XobvfgYJiYA7voHm2vGtOjjVX8gYkwAklFr3RKnlp1xXllLKm9jthgc+38SCTenMjV1O750fQMu+MPZjqBfl6fJqDe0AqpSqsueWbOe7TamsaP4hMQdWwsW3wrCXICDI06XVKhroSqkqmbV6D8tW/cyKBm/Q+ES6FeS9/1juZM7KfTTQlVKVtjTpMN9/PZuv604jRIKQP3wFMVd4uqxaSwNdKVUpX2/OYOu8Z5kZNBtp1AkZPxsatvZ0WbWaBrpS6oJ9/NN26i17kEf8V1HY8TqCbnwXgnRwLU/TQFdKOa3IZuedRSu5cuO9dPdPo6j/YwQNeFjby2sIDXSllFO2HczhiXnreDN7EpEBedhu+pTATiM8XZYqQQNdKXVehcV2Xl+ewts/7ua24J9oLtkwcQG0HeDp0lQpGuhKqXLtzzrDX2dvZHN6DqN7teDRYz+DvRPE9Pd0aaoMGuhKqTIt3JzBY19sxU/gnYkXM7RhBry32epnrm3mNZIGulLqHDNX7WHK4mR6tW7I1HE9aNEwBBY8C4Gh0G2sp8tT5dBAV0r9zufrDzBlcTJDOjfhzQkXE+jvB3nHYet86DYGgsM9XaIqhzPD5yqlaomvtxzikS+2cEVsI14f39MKc4DNc6A4D3rf6dkC1XnpE7pSCoB1KRn8/PnLPB1pZ2z7lgStKTG1wYYPIDoemnX3XIGqQhroSimOnsxn6+x/8X8B8+EU8H3pPQSu0rnfazoNdKVquWKbnec+XsgLtq/I6XAD9W+adu5O4geBwdVfnLogTrWhi8hQEdkpIqki8kg5+4wRkWQRSRKRz1xbplLKXV5aupMxh1+FoFDqj/oPBIWc+0fD3CtU+IQuIv7ANOBqrAmj14vIQmNMcol9YoFHgcuMMcdFRKcoUaqGM8Yw6+e9HF71Ef2CkuGaV6BeY0+XparAmSaXPkCqMSYNQETmAKOA5BL7/AmYZow5DmCMOerqQpVSrlNQbOPxr7bxTeIOVoV8hr3pxfj1ut3TZakqcqbJJRo4UGI53bGupA5ABxFZLSJrRGRoWQcSkUkikigiiZmZmZWrWClVJdmnCxk/fQ2fJ6bzUeulhJmT+I14Ffy0F7O3c+YOlvWOrym1HADEAgOA8cAMEWlwzjcZM90YE2+MiW/cWH+1U6q6GWN4cO5mtmWc5NNrA+hx5Aukz13QvIenS1Mu4EygpwMtSyy3ADLK2GeBMabIGLMH2IkV8EqpGmRuYjordhzl0Wvac9n2ZyGsKQx8zNNlKRdxJtDXA7EiEiMiQcA4YGGpfb4CBgKISCOsJpg0VxaqlKqaA9lnmLI4mb5tI7g18Ds4vAWGPqev8vuQCgPdGFMMTAaWAtuBz40xSSIyRURGOnZbCmSJSDLWKwkPGWOy3FW0UurC2O2Gh+ZtBuCVIZH4ff9/0O4qiLvew5UpV3LqxSJjTAKQUGrdEyW+NsD9jj9KqRrm7R93syYtm/euEprPGwXGDsNe1GFwfYx+rK2Uj1uw6SAvLt3J0222MXjNbeAXAHd8A5HtPF2acjF99V8pH7YmLYuH5m7h0SZrufXwVGh9OYz5EEIbebo05QYa6Er5qJ2Hc5n0USI9GuYzKW+mNW3cxPngH+jp0pSbaJOLUj5ow75sxk7/heBAf2Y1/wqxFcKIVzXMfZwGulI+ZlnyESa8t5YGdQNZPKyIkF1fwRX3a5t5LaBNLkr5iIwTeXyyZh/v/LibrtH1mTmxK5EfDYCItnDZvZ4uT1UDDXSlvNzmAyd4Y0UqK3YcwQAjujXnhes7EvLNA5CdBn/4Uoe/rSU00JXyYku2HuLv/91EWJ0A7u7fjvF9WtEy6DTMvgH2/wL9/wHtBnm6TFVNNNCV8lIf/ryXpxYlcXGrhsy4JZ6GoUFwaDPMmgBnsuDG96HraE+XqaqRBrpSXuj15Sm8smwXV8c14Y3xPQkO9Iet82DBZAiJtF4c0hEUax0NdKW8zI+7Mnll2S5u6BnNizd1xx87fPcUrHoVWvWDMR9BPZ00rDbSQFfKG9ht8Pkt2NN+pFehjaRgCEnzR14Qa1vRaeh1G1z7IgQEebpa5SEa6Ep5gw2zYMdi1oYMYmd+HUZ2jyY0tERwR/fS9nKlga5UjXfqKCx/moMN4hl/+E6eGdWFiH5tPF2VqoE00JWq4fISHiMw/zS35IxlSOemTOzb2tMlqRpKA12pGiq/yMaqZV8yOHkub9n/H6OHXMWfrohBdAxzVQ6nAl1EhgJTAX9ghjHm+VLbbwNeBA46Vr1pjJnhwjqVqjX2HDvNp2v28dWGPcyxPc7hwKYMvetF2jbTidXV+VUY6CLiD0wDrsaaDHq9iCw0xiSX2vW/xpjJbqhRqVrj6y2HuO+/m7AbwyvNltM+KwMzdi6iYa6c4Mxoi32AVGNMmjGmEJgDjHJvWUrVPh+s3sPk2Rvp1qI+a+5uy8icT6HTSKTDNZ4uTXkJZwI9GjhQYjndsa60G0Vki4jME5GWZR1IRCaJSKKIJGZmZlaiXKV8j81ueC5hO08vSubqTk345M4+NFr5uDVV3NDnKz6AUg7OBHpZn8CYUsuLgDbGmG7Ad8CHZR3IGDPdGBNvjIlv3Fh/hVTqcE4+N89Yw7s/pTGxbyventiL4NQESPkWBj4G9ct6dlKqbM58KJoOlHzibgFklNzBGJNVYvE94IWql6aUb1u+/QgPzt3MRUU72BI5k7Ad2cgOoPAMNOkKfe7ydInKyzgT6OuBWBGJwerFMg6YUHIHEWlmjDnkWBwJbHdplUr5kLVpWbyxIpVVqcf4e8Qa7uUdJKg5dBhv7eAXAPF3gL/2KlYXpsKfGGNMsYhMBpZidVucaYxJEpEpQKIxZiHwNxEZCRQD2cBtbqxZKa+UnHGSpxYlsW5PNk1C/VnUfhFd02dD2wEw+gMIifB0icrLiTGlm8OrR3x8vElMTPTIuZWqTgXFNt5ckcrbP+ymft1AHri8EWP3PYH/3p+g71/g6mf0aVw5TUQ2GGPiy9qmP0VKuYkxhuXbj/LCNztIOXqKG3pG8+QlUP+rWyD3EIx6C3re7OkylQ/RQFfKxWx2w7dJh3ljRSrJh07SMqIuH9zWm4FmLXx6F9QJg9uXQIsyH7KUqjQNdKVc5GhuPnMT0/ls7X4OnsgjplEoL93UnVHdmxK46iX44TlrmNuxn0B4c0+Xq3yQBrpSVXD8dCHLko+wZNshVqYco9huuLRdJI8N68TQLk3xt+XD/Ntg+yLoPh5GvAaBwZ4uW/koDXSlLkB+kY1f0rJYtyebtWlZbE7PwWY3RDeoyx2XxzC2d0vaNa732zcsf84K8yH/tj4A1ZESlRtpoCvlBGMMCzdn8MKSHWTk5BPgJ3RrUZ8/92/HkM5N6RIdfu6wtkeS4Zdp0GMi9LvHM4WrWkUDXanzOHGmkLV7snn3x91s3H+Czs3Deeb6LvRrF0lI0Hn++djt8PX91gegV0+pvoJVraaBrlQpNrvh/VVpfLHxIDsO5wLQOKwO/7mxGzf2aoG/nxPNJps/g/2/wMg3IDTSzRUrZdFAV6qEIyfzuXfOJn5Jy6JPmwgeuLoDfWIi6NGqAXUC/J07yOlj8O3j0PISq7lFqWqiga4UVhv50qTDPPblNvIKbfxndDdu6tXiwqd7O7odZo+HwlMw/BXwc2ZAU6VcQwNd1Wp2u+HbZOsloKSMk1zUNIw3J/SkfVTYhR9sRwJ88ScIDIFbF0PTLq4vWKnz0EBXtc7hnHxWpx5j7Z4sft6dRfpx6yWgF0d34/qe0QT6V+Kpeus8mH8nNOsB4z6F+i1cX7hSFdBAV7VGkc3Ouz/u5vXlqRTa7NSvG0ifmAgeGtKREd2aO/dhZ1nOZMOShyE6Hm5bDIF1XVu4Uk7SQFe1wraDOTw8bwvJh04yvFszJg9sT8cmYfhVNsRLWv405J2AW6ZqmCuP0kBXPu3YqQJeWbaLOev2E1mvDu/+oRdDOjd13QkOrIMNs6DfZG0zVx6nga58UpHNzger9/DG8lTyimzc0q8N9w3uQP2QQNedxFYMi++D8GgY8IjrjqtUJTkV6CIyFJiKNWPRDGNMmVORi8hoYC7Q2xijs1cojyjZvDKwY2P+OTyO9lH1zt3RVgxzb7VeAKoMezHk58CYj603QpXysAoDXUT8gWnA1VgTRq8XkYXGmORS+4UBfwPWuqNQpSqSV2jjze9TeOfHNCJCg3hn4sUM7dKs/G9YPwN2LIYuo6Fug8qdtPFF0Om6yn2vUi7mzBN6HyDVGJMGICJzgFFAcqn9ngH+Azzo0gqVqoDd7hg465sdHMrJZ3SvFjw+PO78zSsnD8GKZ6HdVXDjDB0FUfkEZwI9GjhQYjkduKTkDiLSE2hpjFksIuUGuohMAiYBtGrV6sKrVaqEs2ORf7puP5sPnKBLdDhTx/WkT4wTky0vfRRshTDsRQ1z5TOcCfSyftp/nVlaRPyAV4HbKjqQMWY6MB2sSaKdK1EpKLbZ2X4ol5SjuaQcPcXmAydYuycbm93QMqIuL47uxo0Xt3CuG2Lqckj6EgY8BpHt3F+8UtXEmUBPB1qWWG4BZJRYDgO6AD84xr1oCiwUkZH6waiqimKbnVWpx/hm22G+TT5C9ulCAAL8hPZR9bjryrZc26VZ2WORl3RoCyz4i9XMAlCQCxHt4PJ7q+EqlKo+zgT6eiBWRGKAg8A4YMLZjcaYHKDR2WUR+QF4UMNcVcXKlEyeWZzMriOnCA3yZ1CnJgzuFEXn5uG0jgx1/vX8pC/hq79A3YYQN8pa5+cP8XdAQB33XYBSHlBhoBtjikVkMrAUq9viTGNMkohMARKNMQvdXaSqPXYdyeWFJTtYvuMorSJCeH18T66Ja0KwybfCeV8+7HPyYJm7YN270LIvjP0Y6kW5tXalPM2pfujGmAQgodS6J8rZd0DVy1K1TVJGDm+uSGXJtsPUqxPAo9dexG2XtfltDPL598LWzy/8wBffAsNe0qdxVSvom6LKo04VFPPEgm18sfEgYXUC+Oug9txxWQwNQ4N+2yntRyvML7/PmmjZWX4BEOJEjxelfIQGuvKYrek5/HX2RvZnn+Gege2YdGU76tct1Xe8uAC+fgAatoH+/9DBr5Q6Dw10Ve1sdsOMlWm89O1OGtWrw5xJ/crvO776dchKgZvna5grVQENdFWtdh7O5eF5m9mcnsM1cU144cZuv29eKSl7D6x8yeqdEju4egtVygtpoKtqYbMbpn2fyhsrUggPDuSN8T0Z0a1Z+f3HjYGEh6x28KFljgWnlCpFA1253aGcPO6ds4m1e7K5rntznh7ZmYjynsrP2r4QUpfBkOcgvHn1FKqUl9NAV261LPkID8/bTEGxnZdv6s6NvZyYa7MgF5Y8Ak27Qp9J7i9SKR+hga7cIvt0IVMWJfHVpgzimoXz5oSetG1cxpjkZfn+Ocg9BGM+An/9EVXKWfqvRbnU2aFsn1mczMn8Iv5+VSz3DGxPUMB5XtU/eQh2fQMYKDwDa9+BXrdBy97VVbZSPkEDXblEsc3O4i2HePP7VFKPnqJ7i/q8MPoSLmoaXsE3FsJHo+DYzt/W1W8Fg590b8FK+SANdFUlRTY7X/7vIG99n8rerDN0bBLGG+N7MqxrM/ydGcr2lzesMB/9AbS+1FoX3AACg91buFI+SANdXTBjDPuzz/DDzkzeW5lG+vE8OjcP552JvbgmrolzY5IDHN8LP75oTeHW5Qa31qxUbaCBrpy2NT2HD1bv4efdWRw+mQ9Aj5YNmDKqMwM7Rp1/TPLSjIGEh0H8tJ+5Ui6iga4qtGHfcd5YkcIPOzMJCw5gQMco+sRE0DcmgvZR9S4syM/avghSlsI1z0J9J7oyKqUqpIGuynSmsJhFmzP4dO1+tqTnEBEaxENDOnJLv9aEBZ9n8mVnbJ4DC/9m9TO/5G7XFKyU0kBXFmMM2w6eZO2eLNbuyWbN7ixyC4qJjarH0yM7c1N8C0KCqvjjYiuG756EX96ENlfATR+CfxX/56CU+pVT/0JFZCgwFWvGohnGmOdLbb8buAewAaeAScaYZBfXqtzAbjd8m3yEN1akkJRxEoA2kSEM79aMGy5uQe82DSvXpFJa3nGYdyfsXm69/Tnk3xrmSrlYhYEuIv7ANOBqrAmj14vIwlKB/Zkx5h3H/iOBV4ChbqhXuUBhsZ2tB0+wJi2bRZsz2HE4lzaRITx3Q1cGdoyiaX0XdxnM3Amzx8GJA3DdVOulIaWUyznzhN4HSDXGpAGIyBxgFPBroBtjTpbYPxQwrixSVU1BsY2N+05YzSlp2fzvwHHyi+wAxDUL57WxPRjRrRkBzk68fCF2fgPz/2j1K79tMbTq6/pzKKUA5wI9GjhQYjkduKT0TiJyD3A/EAQMKutAIjIJmATQqlWrC61VXYDTBcWsTMlkybbDLN9+lFMFxYhYAT6+TysuiYmkd5uGRNZz01ybxsDKl2HFs9CsG4z7THuzKOVmzgR6WQ2o5zyBG2OmAdNEZALwL+DWMvaZDkwHiI+P16d4Fymy2dmXdZrUo6f434ETrE3LZtvBHIrthoYhgQzv2oyr45rQOybi3Cne3KHwNCy4B5K+hK43wcg3dLYhpaqBM4GeDrQssdwCyDjP/nOAt6tSlDrX0dx81qZls25PNuv3ZnPiTBEABkPWqUKK7db/HwP9he4tGnBX/7Zc1q4RfWIinGtK2fcLfDnJ6olSVUWnIf8kXD0FLv0buOJDVaVUhZwJ9PVArIjEAAeBccCEkjuISKwxJsWxOBxIQVVZxok8ErYe4ptth9mw/zjGQGiQP73aRNC9RYNf92sYGkSHJvWIjQojtkk9ggP9L/xkq1+DglNw0fCqFy4CnW+AdgOrfiyllNMqDHRjTLGITAaWYnVbnGmMSRKRKUCiMWYhMFlEBgNFwHHKaG5RzjtxppDXvkvhkzX7KLYbLmoaxr1XdWBAx8Z0bh7u+g8vT+yHXUvhigfgqsdde2ylVLVxqh+6MSYBSCi17okSX//dxXXVSvlFNmav289r36WQm1/E+D6t+OMVbYlpFOreE2+YZT1Va3dCpbyavilaA+QVWkH+7k+7OXKygMvaR/L4iLiKxxJ3heJC2PgRxA6BBi0r3l8pVWNpoHtIsc3Ouj3ZLNl2mISth8g6XcglMRG8fFMPLmsf6Zq3M52xYxGczoTed1bP+ZRSbqOBXg12HD7JZ2v3s2LHUWyO3iin8ovJLSimbqA/Azo25vbLYugTE1H9xa2fCQ1aQ7urqv/cSimX0kB3k7NjpMxYmUbivuMEBfgxqGMU4XWt/+RBAX5c3r4R/TtEUTeoEr1SKiv3CKR+B8YOBbmwbxUMfgr83PCWqFKqWmmgu5jNbliy7RBvLE9l55FcWkWE8M9hnbixVwsiQoM8W9z+tfDfiXD66G/rgupBj4meq0kp5TIa6C5yNDefz9cfYPa6Axw8kUf7qHruHSPlQm38CBbfb71+P+47CGtqrQ8Oh+D6nq1NKeUSGuhVdDQ3n1e+3cW8DekU242jh0onrolr6vzcmpV1JhsW/hUyNp1/P2OH3AxoOxBGz4QQD7TVK6XcTgO9kvKLbMxcvYdpK1IptNmZ2Lc1t/RrTdvG9aqngCNJMHs85B6Gzv8P/Cq4lY07QN97wF9vuVK+Sv91X6DSfcavjmvCY8M6uf/ln5KSF8KXd0OdMLg9AVrEV9+5lVI1lga6k+x2wydr9/H68hSOnSqkT0wEr47twaXtGlVnEfDjC/Dj8xAdD2M/gfBm1RRoCNQAAA3mSURBVHd+pVSNpoHuhNSjp/jH/C1s2Hecfm0jmTYhlkvaRlZvEQW51lP5jsXQfQKMeNWaNEIppRw00M8jJ6+Imav28PaPuwkJ8ufVsd25vkd09b3FeVZxAXwwDI5sgyHPQd8/65C0SqlzaKCX4fjpQt5ftYcPf95LbkExI7o148nrOtM4zE2z+1Rk9VQ4vAXGfAxxIz1Tg1KqxtNAL+VkfhHXvbmK9ON5XNulKZMHtadzcw/2087aDT+9ZPVk0TBXSp2HBnopUxYlk3Eij/9O6lv97eSlGQMJD4F/kNXUopRS51EDXmGsOZYlH2HehnT+MqC958McIPkr2L0cBv1Le7MopSrkVKCLyFAR2SkiqSLySBnb7xeRZBHZIiLLRaS160t1r6xTBTz6xRbimoXzt6tiPV0OHN0OS/4BTbtB7z96uhqllBeoMNBFxB+YBlwLxAHjRSSu1G7/A+KNMd2AecB/XF2oO50uKObheVvIySvilbHdCQrw8C8uO76GGYOtJpfr39a3O5VSTnEmKfoAqcaYNAARmQOMApLP7mCM+b7E/msArxm+b2VKJo9+sZX043k8dV01zRJUkjHWcLanM63lo9vh59eheU8Y+ynUj67eepRSXsuZQI8GDpRYTgcuOc/+dwJLytogIpOASQCtWrVyskT3yC+y8dTCJOasP0DbRqHMvbsfvdt4YNCqDbNg8b2/X9dtLFw3FQLrVn89Simv5Uygl/UGiylzR5GJQDzQv6ztxpjpwHSA+Pj4Mo9RHTJO5DHp40SSMk5yV/+23De4A8GB1TjJxFmnMuG7J6H15XD9NGudfxCEN6/+WpRSXs+ZQE8HSs4e3ALIKL2TiAwG/gn0N8YUuKY810vcm83dn2wgv8jOe3+IZ3BcE88Vs+xxKDwDI16Bhm08V4dSyic4E+jrgVgRiQEOAuOACSV3EJGewLvAUGPM0XMP4XkFxTamrUjlrR9206JhXeZMiqd9VJjnCtqzEjbPhisegMYdPVeHUspnVBjoxphiEZkMLAX8gZnGmCQRmQIkGmMWAi8C9YC5jnFO9htjasxrjRv3H+fheVtIPXqKG3pG8+R1nakfEuj+E+fnQMoysBWdu23VK9CgFVzxoPvrUErVCk71hzPGJAAJpdY9UeLrwS6uy2U+XbuPx7/aRtPwYD64vTcDO0ZVz4kzd8Gc8ZCVWvZ2vwAYPweCQqqnHqWUz/PZDs7GGF5dtovXV6QysGNjXh/fk7DgangqB9i1FOb/0fqA8+b5ENnu3H3qhEFoNY6lrpTyeT4Z6Da74Z9fbmXO+gOMiW/Bv/9fV/dN1Gy3wQ/Pw9a5gLH6lZ/YD027wrjPoEHLCg+hlFKu4JOB/smafcxZf4B7BrbjwWs6um/88vwc60k85VtoNwhCG1vru9wIVz6kzSlKqWrlc4GefbqQl7/dyWXtI90b5sdSYfY4OL4Hhr+s460opTzO5wL95W93crrQxpPXdXZfmKd8B/PusMZYuWUBtLncPedRSqkL4FPD5247mMNn6/ZzS7/WdGjihj7mxlizB3062upy+KfvNcyVUjWGzzyhG2N4elESDUOCuHdwB9efoCgPFv4Ntn4OcdfD9W9BUKjrz6OUUpXkE4GeV2jj6UVJrN97nOdv6Er9ui7unphzEOZMgEObrMkmrnhQJ2lWStU4Xh/oOw/nMvmzjaRmnuLPA9oxJt7F3QQPrLfCvCgPxs2Gi4a59vhKKeUiXh3oCzYd5OF5WwgLDuSjO/pwRWxj157gTDbMHmu9BHTrIoi6yLXHV0opF/LaQH/vpzT+L2E7fWIimDbhYhqH1XH9Sb57CvJOwC0LNcyVUjWe1wW63W74d8J2Zqzaw7CuTXllTA/3jGV+YB1s/BD6TYamXVx/fKWUcjGvC/Spy1OYsWoPt13ahsdHxOHv54YPJ23FsPg+CI+GAY+6/vhKKeUGXhfoE/u2plFYHSZe0sp9Lw6tfRuObIMxH0Odeu45h1JKuZjXBXrjsDr8oW9r9xzcGPhlGix7AjoMhU7Xuec8SinlBl4X6G5TlG9N1rx5NnQaCde/rX3NlVJexalX/0VkqIjsFJFUEXmkjO1XishGESkWkdGuL9PNjLFe5988Gwb+E276UJtalFJep8JAFxF/YBpwLRAHjBeRuFK77QduAz5zdYHVYt9q2LsShr4A/R8GP58a4kYpVUs40+TSB0g1xqQBiMgcYBSQfHYHY8xexza7G2p0v/XvQ3B96HWrpytRSqlKc+ZRNBo4UGI53bHugonIJBFJFJHEzMzMyhzC9U4dhe2LoMfNEFjX09UopVSlORPoZX0yaCpzMmPMdGNMvDEmvnFjF7+mX1n/+xjsRRB/h6crUUqpKnEm0NOBkiNetQAy3FNONbPbIHEWxFwJjWI9XY1SSlWJM4G+HogVkRgRCQLGAQvdW1Y1Sf0OcvZD/J2erkQppaqswkA3xhQDk4GlwHbgc2NMkohMEZGRACLSW0TSgZuAd0UkyZ1Fu8z696FeE7houKcrUUqpKnPqxSJjTAKQUGrdEyW+Xo/VFOM9Dm6AlG/hyofA38UTYiillAfUzg7XtmJYdC+ENYVL/+rpapRSyiVq56v/62fA4S1w0ywIDvd0NUop5RK17wn95CFY8Sy0u8qa7FkppXxE7Qv0pY+CrRCGv6SDbymlfErtCvTU5ZD0JVz5IES09XQ1SinlUrUn0IvyIeFBiGwPl/3d09UopZTL1Z4PRVe9CtlpcMsCCHDDhNJKKeVhteMJ/VgqrHoFuoyGtgM8XY1SSrmF9z2hZ+6y5vs8KyQCYvqX/wGnMZDwAAQEw5B/V0+NSinlAd4X6LuWWHN+ltR9PIx4DQKDf7++8DR89RdI+wGGvQRhTaqtTKWUqm7eF+g9JkLskN+Wk7+CH56DzJ0w7lMIb26tP74P5twMR5Pg6meg9x89U69SSlUT7wv00Ejrz1lRj0CTLvDlXfBWXwhrZq0/mQEITJgLsYM9UqpSSlUn7wv0snQaARHLYPVUKM6z1jXvCVc8CI3ae7Y2pZSqJr4R6ABN4uCGdz1dhVJKeUzt6LaolFK1gAa6Ukr5CKcCXUSGishOEUkVkUfK2F5HRP7r2L5WRNq4ulCllFLnV2Ggi4g/MA24FogDxotIXKnd7gSOG2PaA68CL7i6UKWUUufnzBN6HyDVGJNmjCkE5gCjSu0zCvjQ8fU84CoRHZtWKaWqkzOBHg0cKLGc7lhX5j6OSaVzgMhS+yAik0QkUUQSMzMzK1exUkqpMjkT6GU9aZtK7IMxZroxJt4YE9+4cWNn6lNKKeUkZwI9HWhZYrkFkFHePiISANQHsl1RoFJKKec482LReiBWRGKAg8A4YEKpfRYCtwK/AKOBFcaYc57QS9qwYcMxEdl34SUD0Ag4Vsnv9Wa18bpr4zVD7bzu2njNcOHX3bq8DRUGujGmWEQmA0sBf2CmMSZJRKYAicaYhcD7wMcikor1ZD7OieNWus1FRBKNMfGV/X5vVRuvuzZeM9TO666N1wyuvW6nXv03xiQACaXWPVHi63zgJlcUpJRSqnL0TVGllPIR3hro0z1dgIfUxuuujdcMtfO6a+M1gwuvWyr47FIppZSX8NYndKWUUqVooCullI/wukCvaORHXyAiLUXkexHZLiJJIvJ3x/oIEVkmIimOvxt6ulZXExF/EfmfiCx2LMc4RvBMcYzoGeTpGl1NRBqIyDwR2eG45/1qyb2+z/HzvU1EZotIsK/dbxGZKSJHRWRbiXVl3luxvO7Iti0icvGFns+rAt3JkR99QTHwgDGmE9AXuMdxnY8Ay40xscByx7Kv+TuwvcTyC8Crjms+jjWyp6+ZCnxjjLkI6I51/T59r0UkGvgbEG+M6YL1jss4fO9+zwKGllpX3r29Foh1/JkEvH2hJ/OqQMe5kR+9njHmkDFmo+PrXKx/4NH8flTLD4HrPVOhe4hIC2A4MMOxLMAgrBE8wTevORy4EuvlPIwxhcaYE/j4vXYIAOo6hgsJAQ7hY/fbGPMT5w6DUt69HQV8ZCxrgAYi0uxCzudtge7MyI8+xTFZSE9gLdDEGHMIrNAHojxXmVu8BjwM2B3LkcAJxwie4Jv3uy2QCXzgaGqaISKh+Pi9NsYcBF4C9mMFeQ6wAd+/31D+va1yvnlboDs1qqOvEJF6wHzgXmPMSU/X404iMgI4aozZUHJ1Gbv62v0OAC4G3jbG9ARO42PNK2VxtBuPAmKA5kAoVpNDab52v8+nyj/v3hbozoz86BNEJBArzD81xnzhWH3k7K9gjr+Peqo+N7gMGCkie7Ga0gZhPbE3cPxKDr55v9OBdGPMWsfyPKyA9+V7DTAY2GOMyTTGFAFfAJfi+/cbyr+3Vc43bwv0X0d+dHz6PQ5rpEef4mg7fh/Ybox5pcSms6Na4vh7QXXX5i7GmEeNMS2MMW2w7usKY8zNwPdYI3iCj10zgDHmMHBARDo6Vl0FJOPD99phP9BXREIcP+9nr9un77dDefd2IXCLo7dLXyDnbNOM04wxXvUHGAbsAnYD//R0PW66xsuxftXaAmxy/BmG1aa8HEhx/B3h6VrddP0DgMWOr9sC64BUYC5Qx9P1ueF6ewCJjvv9FdCwNtxr4GlgB7AN+Bio42v3G5iN9RlBEdYT+J3l3VusJpdpjmzbitUD6ILOp6/+K6WUj/C2JhellFLl0EBXSikfoYGulFI+QgNdKaV8hAa6Ukr5CA10pZTyERroSinlI/4/PVuFly3fEMUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores2)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pytorch没有实现L1正则化，从网上抄了一段。  \n",
    "一开始alpha太大，出现非常神奇的效果。  \n",
    "后来调小了之后结果就正常了。  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
